findEthics/Atlas
0
1"""2Performance tests for the Atlas AI Chat API3 4Consolidated from:5- test_performance_user_auth.py6- Performance portions of other test files7"""8 9import asyncio10from datetime import datetime, timedelta11import os12import time13 14import psutil15import pytest16import random17 18from analytics.collectors import create_session, track_message, track_search19from analytics.dashboard import get_authenticated_vs_anonymous_metrics, get_basic_stats, get_hourly_message_stats, get_user_analytics, get_user_statistics20from analytics.database import get_messages_collection, get_sessions_collection21from tests.utilities import MockDataGenerator, PerformanceHelpers, TestHelpers, create_test_user_data, skip_if_no_database22 23 TestHelpers, PerformanceHelpers, MockDataGenerator,24 skip_if_no_database, create_test_user_data25)26 27 28class TestDatabaseIndexPerformance:29 """Test performance of database indexes for user_id queries"""30 31 @pytest.mark.asyncio32 @skip_if_no_database()33 async def test_user_id_index_performance(self):34 """Test performance of user_id index queries"""35 # Create test data with multiple users36 user_ids = [f"perf_user_{i}" for i in range(10)]37 sessions_per_user = 238 messages_per_session = 339 40 # Create test data41 start_time = time.time()42 43 for user_id in user_ids:44 await create_test_user_data(user_id, sessions_per_user, messages_per_session)45 46 data_creation_time = time.time() - start_time47 48 # Wait for data to be persisted49 await TestHelpers.wait_for_data_persistence(2.0)50 51 # Test query performance52 sessions_collection = await get_sessions_collection()53 messages_collection = await get_messages_collection()54 55 if sessions_collection and messages_collection:56 # Test individual user queries57 start_time = time.time()58 59 for user_id in user_ids:60 user_sessions = await sessions_collection.count_documents({"user_id": user_id})61 assert user_sessions >= sessions_per_user62 63 user_messages = await messages_collection.count_documents({"user_id": user_id})64 assert user_messages >= sessions_per_user * messages_per_session65 66 individual_query_time = time.time() - start_time67 avg_query_time = individual_query_time / len(user_ids)68 69 # Performance assertion70 assert avg_query_time < 0.5, f"Individual queries too slow: {avg_query_time:.4f}s average"71 72 # Test bulk queries73 start_time = time.time()74 75 # Query all authenticated sessions76 auth_sessions = await sessions_collection.count_documents({"user_id": {"$ne": None}})77 78 # Query all authenticated messages79 auth_messages = await messages_collection.count_documents({"user_id": {"$ne": None}})80 81 bulk_query_time = time.time() - start_time82 83 # Performance assertion84 assert bulk_query_time < 3.0, f"Bulk queries too slow: {bulk_query_time:.2f}s"85 86 # Verify we got expected results87 assert auth_sessions >= len(user_ids) * sessions_per_user88 assert auth_messages >= len(user_ids) * sessions_per_user * messages_per_session89 90 @pytest.mark.asyncio91 @skip_if_no_database()92 async def test_compound_index_performance(self):93 """Test performance of compound (user_id, timestamp) index queries"""94 # Create test data with timestamps spread over time95 user_id = "compound_perf_user"96 session = await create_session(user_id=user_id)97 98 num_messages = 2099 100 for i in range(num_messages):101 await track_message(102 session_id=session.session_id,103 prompt_length=50,104 response_length=100,105 response_time_ms=1000,106 user_id=user_id107 )108 109 # Small delay to ensure different timestamps110 await asyncio.sleep(0.01)111 112 # Wait for data to be persisted113 await TestHelpers.wait_for_data_persistence(2.0)114 115 # Test compound queries116 messages_collection = await get_messages_collection()117 118 if messages_collection:119 # Test various time range queries120 time_ranges = [121 ("1 hour", timedelta(hours=1)),122 ("6 hours", timedelta(hours=6)),123 ("24 hours", timedelta(hours=24))124 ]125 126 for range_name, time_delta in time_ranges:127 start_time = time.time()128 cutoff_time = datetime.utcnow() - time_delta129 130 recent_messages = await messages_collection.count_documents({131 "user_id": user_id,132 "timestamp": {"$gte": cutoff_time}133 })134 135 query_time = time.time() - start_time136 137 # Performance assertion138 assert query_time < 1.0, f"{range_name} query too slow: {query_time:.4f}s"139 140 # Should find our messages141 assert recent_messages >= num_messages142 143 @pytest.mark.asyncio144 @skip_if_no_database()145 async def test_sparse_index_performance(self):146 """Test performance of sparse indexes with mixed null/non-null user_id values"""147 # Create mixed data (authenticated and anonymous)148 num_auth_users = 5149 num_anon_sessions = 10150 messages_per_session = 3151 152 # Create authenticated user data153 for i in range(num_auth_users):154 user_id = f"sparse_user_{i}"155 await create_test_user_data(user_id, 1, messages_per_session)156 157 # Create anonymous user data158 for i in range(num_anon_sessions):159 session = await create_session(user_id=None)160 161 for j in range(messages_per_session):162 await track_message(163 session_id=session.session_id,164 prompt_length=50,165 response_length=100,166 response_time_ms=1000,167 user_id=None168 )169 170 # Wait for data to be persisted171 await TestHelpers.wait_for_data_persistence(2.0)172 173 # Test sparse index queries174 sessions_collection = await get_sessions_collection()175 messages_collection = await get_messages_collection()176 177 if sessions_collection and messages_collection:178 # Test authenticated user queries179 start_time = time.time()180 auth_session_count = await sessions_collection.count_documents({"user_id": {"$ne": None}})181 auth_query_time = time.time() - start_time182 183 # Test anonymous user queries184 start_time = time.time()185 anon_session_count = await sessions_collection.count_documents({"user_id": None})186 anon_query_time = time.time() - start_time187 188 # Test specific user queries189 start_time = time.time()190 specific_user_sessions = await sessions_collection.count_documents({"user_id": "sparse_user_0"})191 specific_query_time = time.time() - start_time192 193 # Performance assertions194 assert auth_query_time < 1.0, f"Auth query too slow: {auth_query_time:.4f}s"195 assert anon_query_time < 1.0, f"Anon query too slow: {anon_query_time:.4f}s"196 assert specific_query_time < 0.5, f"Specific query too slow: {specific_query_time:.4f}s"197 198 # Verify results199 assert auth_session_count >= num_auth_users200 assert anon_session_count >= num_anon_sessions201 assert specific_user_sessions >= 1202 203 204class TestAnalyticsFunctionPerformance:205 """Test performance of analytics functions with user authentication"""206 207 @pytest.mark.asyncio208 async def test_basic_stats_performance(self):209 """Test performance of get_basic_stats function"""210 # Create some test data211 await self._create_performance_test_data()212 213 # Test get_basic_stats performance214 start_time = time.time()215 stats = await get_basic_stats()216 stats_time = time.time() - start_time217 218 assert isinstance(stats, dict)219 assert "total_sessions" in stats220 assert "total_messages" in stats221 222 # Performance assertion223 assert stats_time < 5.0, f"Basic stats too slow: {stats_time:.4f}s"224 225 @pytest.mark.asyncio226 async def test_user_statistics_performance(self):227 """Test performance of get_user_statistics function"""228 # Create test data229 await self._create_performance_test_data()230 231 # Test get_user_statistics performance232 start_time = time.time()233 user_stats = await get_user_statistics()234 stats_time = time.time() - start_time235 236 assert isinstance(user_stats, dict)237 assert "unique_authenticated_users" in user_stats238 assert "authenticated_sessions" in user_stats239 240 # Performance assertion241 assert stats_time < 8.0, f"User statistics too slow: {stats_time:.4f}s"242 243 @pytest.mark.asyncio244 async def test_user_analytics_performance(self):245 """Test performance of get_user_analytics function"""246 # Create test user with substantial data247 user_id = "analytics_perf_user"248 await create_test_user_data(user_id, num_sessions=2, messages_per_session=10)249 250 # Wait for data to be persisted251 await TestHelpers.wait_for_data_persistence()252 253 # Test get_user_analytics performance254 start_time = time.time()255 user_analytics = await get_user_analytics(user_id)256 analytics_time = time.time() - start_time257 258 assert isinstance(user_analytics, dict)259 assert user_analytics.get("user_id") == user_id260 261 # Performance assertion262 assert analytics_time < 5.0, f"User analytics too slow: {analytics_time:.4f}s"263 264 @pytest.mark.asyncio265 async def test_comparison_metrics_performance(self):266 """Test performance of get_authenticated_vs_anonymous_metrics function"""267 # Create mixed test data268 await self._create_performance_test_data()269 270 # Test get_authenticated_vs_anonymous_metrics performance271 start_time = time.time()272 comparison_metrics = await get_authenticated_vs_anonymous_metrics()273 comparison_time = time.time() - start_time274 275 assert isinstance(comparison_metrics, dict)276 assert "authenticated" in comparison_metrics277 assert "anonymous" in comparison_metrics278 279 # Performance assertion280 assert comparison_time < 8.0, f"Comparison metrics too slow: {comparison_time:.4f}s"281 282 @pytest.mark.asyncio283 async def test_hourly_stats_performance(self):284 """Test performance of get_hourly_message_stats function"""285 # Create test data286 await self._create_performance_test_data()287 288 # Test hourly stats performance289 start_time = time.time()290 hourly_stats = await get_hourly_message_stats(hours=24)291 hourly_time = time.time() - start_time292 293 assert isinstance(hourly_stats, list)294 assert len(hourly_stats) == 24295 296 # Performance assertion297 assert hourly_time < 5.0, f"Hourly stats too slow: {hourly_time:.4f}s"298 299 async def _create_performance_test_data(self):300 """Create test data for performance testing"""301 # Create authenticated users302 for i in range(3):303 user_id = f"perf_test_user_{i}"304 await create_test_user_data(user_id, num_sessions=1, messages_per_session=5)305 306 # Create anonymous users307 for i in range(2):308 session = await create_session(user_id=None)309 310 # Create messages for anonymous users311 for j in range(3):312 await track_message(313 session_id=session.session_id,314 prompt_length=random.randint(20, 100),315 response_length=random.randint(50, 200),316 response_time_ms=random.randint(500, 2000),317 used_search=random.choice([True, False]),318 user_id=None319 )320 321 322class TestConcurrentUserPerformance:323 """Test performance with concurrent user operations"""324 325 @pytest.mark.asyncio326 async def test_concurrent_user_creation(self):327 """Test performance of concurrent user session creation"""328 num_concurrent_users = 10329 330 async def create_user_session(user_id: str):331 session = await create_session(user_id=user_id)332 333 # Create a few messages for each user334 for i in range(2):335 await track_message(336 session_id=session.session_id,337 prompt_length=50,338 response_length=100,339 response_time_ms=1000,340 user_id=user_id341 )342 343 return session344 345 # Create concurrent tasks346 start_time = time.time()347 tasks = [348 create_user_session(f"concurrent_user_{i}")349 for i in range(num_concurrent_users)350 ]351 352 sessions = await asyncio.gather(*tasks)353 concurrent_time = time.time() - start_time354 355 assert len(sessions) == num_concurrent_users356 357 # Performance assertion358 avg_time_per_user = concurrent_time / num_concurrent_users359 assert avg_time_per_user < 2.0, f"Concurrent creation too slow: {avg_time_per_user:.2f}s per user"360 361 @pytest.mark.asyncio362 async def test_concurrent_user_queries(self):363 """Test performance of concurrent user-specific queries"""364 # Create test users first365 user_ids = [f"query_user_{i}" for i in range(5)]366 367 for user_id in user_ids:368 await create_test_user_data(user_id, num_sessions=1, messages_per_session=2)369 370 # Wait for data to be persisted371 await TestHelpers.wait_for_data_persistence()372 373 # Test concurrent queries374 async def query_user_analytics(user_id: str):375 return await get_user_analytics(user_id)376 377 start_time = time.time()378 tasks = [query_user_analytics(user_id) for user_id in user_ids]379 results = await asyncio.gather(*tasks)380 concurrent_query_time = time.time() - start_time381 382 assert len(results) == len(user_ids)383 for i, result in enumerate(results):384 assert result.get("user_id") == user_ids[i]385 386 # Performance assertion387 avg_query_time = concurrent_query_time / len(user_ids)388 assert avg_query_time < 2.0, f"Concurrent queries too slow: {avg_query_time:.2f}s per query"389 390 @pytest.mark.asyncio391 async def test_concurrent_mixed_operations(self):392 """Test performance of mixed concurrent operations"""393 # Define different types of operations394 async def create_user_data(user_id: str):395 session = await create_session(user_id=user_id)396 await track_message(397 session_id=session.session_id,398 prompt_length=50,399 response_length=100,400 response_time_ms=1000,401 user_id=user_id402 )403 return f"created_{user_id}"404 405 async def query_basic_stats():406 stats = await get_basic_stats()407 return f"stats_{stats['total_sessions']}"408 409 async def query_user_stats():410 stats = await get_user_statistics()411 return f"user_stats_{stats['total_sessions']}"412 413 # Create mixed operations414 operations = []415 416 # Add user creation operations417 for i in range(3):418 operations.append(create_user_data(f"mixed_user_{i}"))419 420 # Add query operations421 operations.append(query_basic_stats())422 operations.append(query_user_stats())423 424 # Execute concurrently425 start_time = time.time()426 results = await asyncio.gather(*operations)427 total_time = time.time() - start_time428 429 assert len(results) == len(operations)430 431 # Performance assertion432 avg_operation_time = total_time / len(operations)433 assert avg_operation_time < 3.0, f"Mixed operations too slow: {avg_operation_time:.2f}s per operation"434 435 436class TestMemoryPerformance:437 """Test memory usage with user authentication"""438 439 @pytest.mark.asyncio440 async def test_memory_usage_with_users(self):441 """Test that user_id fields don't significantly increase memory usage"""442 # Get initial memory usage443 process = psutil.Process(os.getpid())444 initial_memory = process.memory_info().rss / 1024 / 1024 # MB445 446 # Create substantial amount of data447 num_users = 10448 messages_per_user = 5449 450 for i in range(num_users):451 user_id = f"memory_test_user_{i}"452 await create_test_user_data(user_id, num_sessions=1, messages_per_session=messages_per_user)453 454 # Get final memory usage455 final_memory = process.memory_info().rss / 1024 / 1024 # MB456 memory_increase = final_memory - initial_memory457 458 # Memory increase should be reasonable459 total_records = num_users * (1 + messages_per_user) # sessions + messages460 memory_per_record = memory_increase / total_records if total_records > 0 else 0461 462 # Performance assertion (should be less than 2MB per record)463 assert memory_per_record < 2.0, f"Memory usage too high: {memory_per_record:.3f}MB per record"464 465 @pytest.mark.asyncio466 async def test_memory_usage_with_large_dataset(self):467 """Test memory usage with larger dataset"""468 # Get initial memory usage469 process = psutil.Process(os.getpid())470 initial_memory = process.memory_info().rss / 1024 / 1024 # MB471 472 # Create larger dataset473 num_users = 20474 475 for i in range(num_users):476 user_id = f"large_memory_test_user_{i}"477 session = await create_session(user_id=user_id)478 479 # Create multiple messages per user480 for j in range(3):481 await track_message(482 session_id=session.session_id,483 prompt_length=random.randint(50, 200),484 response_length=random.randint(100, 500),485 response_time_ms=random.randint(500, 3000),486 user_id=user_id487 )488 489 # Get final memory usage490 final_memory = process.memory_info().rss / 1024 / 1024 # MB491 memory_increase = final_memory - initial_memory492 493 # Memory increase should be reasonable for the amount of data494 total_records = num_users * 4 # 1 session + 3 messages per user495 memory_per_record = memory_increase / total_records if total_records > 0 else 0496 497 # Performance assertion498 assert memory_per_record < 3.0, f"Large dataset memory usage too high: {memory_per_record:.3f}MB per record"499 500 501class TestScalabilityPerformance:502 """Test scalability with increasing data volumes"""503 504 @pytest.mark.asyncio505 @skip_if_no_database()506 async def test_query_performance_with_scale(self):507 """Test that query performance doesn't degrade significantly with more data"""508 # Create baseline data and measure performance509 baseline_user = "scale_baseline_user"510 await create_test_user_data(baseline_user, num_sessions=1, messages_per_session=5)511 512 # Wait for data to be persisted513 await TestHelpers.wait_for_data_persistence()514 515 # Measure baseline query performance516 start_time = time.time()517 baseline_analytics = await get_user_analytics(baseline_user)518 baseline_time = time.time() - start_time519 520 # Create more data (simulate scale)521 for i in range(5):522 scale_user = f"scale_user_{i}"523 await create_test_user_data(scale_user, num_sessions=2, messages_per_session=10)524 525 # Wait for data to be persisted526 await TestHelpers.wait_for_data_persistence()527 528 # Measure performance with more data529 start_time = time.time()530 scaled_analytics = await get_user_analytics(baseline_user)531 scaled_time = time.time() - start_time532 533 # Performance should not degrade significantly534 performance_ratio = scaled_time / baseline_time if baseline_time > 0 else 1535 assert performance_ratio < 3.0, f"Performance degraded too much: {performance_ratio:.2f}x slower"536 537 # Results should be consistent538 assert baseline_analytics["user_id"] == scaled_analytics["user_id"]539 assert baseline_analytics["total_sessions"] == scaled_analytics["total_sessions"]540 541 @pytest.mark.asyncio542 async def test_analytics_performance_with_scale(self):543 """Test analytics function performance with increasing data"""544 # Measure performance with small dataset545 small_users = 2546 for i in range(small_users):547 user_id = f"small_scale_user_{i}"548 await create_test_user_data(user_id, num_sessions=1, messages_per_session=2)549 550 start_time = time.time()551 small_stats = await get_user_statistics()552 small_time = time.time() - start_time553 554 # Add more data555 additional_users = 5556 for i in range(additional_users):557 user_id = f"large_scale_user_{i}"558 await create_test_user_data(user_id, num_sessions=1, messages_per_session=3)559 560 # Measure performance with larger dataset561 start_time = time.time()562 large_stats = await get_user_statistics()563 large_time = time.time() - start_time564 565 # Performance should scale reasonably566 data_ratio = (small_users + additional_users) / small_users567 performance_ratio = large_time / small_time if small_time > 0 else 1568 569 # Performance should not degrade more than linearly with data size570 assert performance_ratio < data_ratio * 2, f"Performance scaling too poor: {performance_ratio:.2f}x for {data_ratio:.2f}x data"571 572 # Results should reflect the additional data573 assert large_stats["unique_authenticated_users"] >= small_stats["unique_authenticated_users"]574 assert large_stats["total_sessions"] >= small_stats["total_sessions"]575 576 577class TestPerformanceBenchmarks:578 """Benchmark tests for performance regression detection"""579 580 @pytest.mark.asyncio581 async def test_user_creation_benchmark(self):582 """Benchmark user creation performance"""583 num_iterations = 10584 times = []585 586 for i in range(num_iterations):587 user_id = f"benchmark_user_{i}"588 589 start_time = time.time()590 session = await create_session(user_id=user_id)591 await track_message(592 session_id=session.session_id,593 prompt_length=50,594 response_length=100,595 response_time_ms=1000,596 user_id=user_id597 )598 end_time = time.time()599 600 times.append(end_time - start_time)601 602 # Calculate statistics603 avg_time = sum(times) / len(times)604 max_time = max(times)605 min_time = min(times)606 607 # Benchmark assertions608 assert avg_time < 1.0, f"Average user creation too slow: {avg_time:.3f}s"609 assert max_time < 3.0, f"Worst case user creation too slow: {max_time:.3f}s"610 assert min_time < 0.5, f"Best case user creation too slow: {min_time:.3f}s"611 612 @pytest.mark.asyncio613 async def test_analytics_query_benchmark(self):614 """Benchmark analytics query performance"""615 # Create test data616 for i in range(3):617 user_id = f"analytics_benchmark_user_{i}"618 await create_test_user_data(user_id, num_sessions=1, messages_per_session=3)619 620 # Wait for data to be persisted621 await TestHelpers.wait_for_data_persistence()622 623 # Benchmark different analytics functions624 functions_to_test = [625 ("basic_stats", get_basic_stats),626 ("user_statistics", get_user_statistics),627 ]628 629 for func_name, func in functions_to_test:630 times = []631 632 # Run multiple iterations633 for i in range(5):634 start_time = time.time()635 result = await func()636 end_time = time.time()637 638 times.append(end_time - start_time)639 assert isinstance(result, dict) # Verify function works640 641 # Calculate statistics642 avg_time = sum(times) / len(times)643 max_time = max(times)644 645 # Benchmark assertions646 assert avg_time < 3.0, f"{func_name} average too slow: {avg_time:.3f}s"647 assert max_time < 8.0, f"{func_name} worst case too slow: {max_time:.3f}s"648 649 650if __name__ == "__main__":651 # Run tests manually for debugging652 async def run_basic_tests():653 test_index = TestDatabaseIndexPerformance()654 print("✅ Database index performance tests defined")655 656 test_analytics = TestAnalyticsFunctionPerformance()657 await test_analytics.test_basic_stats_performance()658 print("✅ Analytics function performance tests passed")659 660 test_concurrent = TestConcurrentUserPerformance()661 print("✅ Concurrent user performance tests defined")662 663 asyncio.run(run_basic_tests())