findEthics/Atlas
0
1#!/usr/bin/env python32"""3Regression testing for search optimizer refactoring.4 5This script validates that the refactored functions behave exactly the same6as they did before the refactoring, ensuring no behavioral changes.7"""8 9import sys10from typing import List, Dict, Any, Optional11 12def test_search_decision_consistency():13 """Test that search decisions are consistent and logical"""14 15 print("🔄 Regression Testing: Search Decision Consistency")16 print("=" * 60)17 18 try:19 from search_optimizer import should_perform_search20 21 # Test cases with expected behaviors that should remain consistent22 test_cases = [23 {24 "name": "Greeting detection",25 "prompt": "Hello there!",26 "history": None,27 "expected_decision": False,28 "expected_reason_contains": "greeting"29 },30 {31 "name": "New information request",32 "prompt": "What is the latest news about artificial intelligence?",33 "history": None,34 "expected_decision": True,35 "expected_reason_contains": ["information", "history", "No conversation"]36 },37 {38 "name": "Follow-up elaboration",39 "prompt": "Tell me more about that",40 "history": [{"user": "What is AI?", "assistant": "AI is artificial intelligence used to create smart systems..."}],41 "expected_decision": False,42 "expected_reason_contains": "Follow-up"43 },44 {45 "name": "Referential question",46 "prompt": "Can you explain that concept better?",47 "history": [{"user": "What is ML?", "assistant": "Machine learning is a subset of AI that enables systems to learn..."}],48 "expected_decision": False,49 "expected_reason_contains": "question"50 },51 {52 "name": "Continuation request",53 "prompt": "What else should I know?",54 "history": [{"user": "Basics of AI?", "assistant": "AI involves creating intelligent systems..."}],55 "expected_decision": True,56 "expected_reason_contains": ["topic", "patterns", "insufficient"]57 },58 {59 "name": "Fresh topic change",60 "prompt": "How does quantum computing work?",61 "history": [{"user": "What is AI?", "assistant": "AI is artificial intelligence..."}],62 "expected_decision": True,63 "expected_reason_contains": ["information", "topic"]64 }65 ]66 67 passed_tests = 068 total_tests = len(test_cases)69 70 for i, case in enumerate(test_cases, 1):71 print(f"\n🔍 Test {i}: {case['name']}")72 73 result = should_perform_search(74 case["prompt"],75 case["history"]76 )77 78 # Check decision consistency79 decision_correct = result["should_search"] == case["expected_decision"]80 81 # Check reason consistency82 reason_correct = False83 expected_reasons = case["expected_reason_contains"]84 if isinstance(expected_reasons, str):85 expected_reasons = [expected_reasons]86 87 for expected_reason in expected_reasons:88 if expected_reason.lower() in result["reason"].lower():89 reason_correct = True90 break91 92 print(f" 📝 Decision: {result['should_search']} (expected: {case['expected_decision']})")93 print(f" 📝 Reason: {result['reason']}")94 print(f" 📊 Confidence: {result['confidence']:.2f}")95 96 if decision_correct and reason_correct:97 print(" ✅ PASS - Behavior consistent")98 passed_tests += 199 else:100 print(" ❌ FAIL - Behavior inconsistent")101 if not decision_correct:102 print(" 🔸 Decision mismatch")103 if not reason_correct:104 print(" 🔸 Reason doesn't match expected pattern")105 106 print(f"\n📊 Search Decision Tests: {passed_tests}/{total_tests} passed")107 return passed_tests == total_tests108 109 except Exception as e:110 print(f"❌ Search decision regression test error: {e}")111 return False112 113def test_conversation_history_consistency():114 """Test conversation history analysis consistency"""115 116 print("\n📊 Regression Testing: Conversation History Analysis")117 print("=" * 60)118 119 try:120 from search_optimizer import has_meaningful_conversation_history121 122 # Test cases with expected behaviors123 test_cases = [124 {125 "name": "None history",126 "history": None,127 "expected": False128 },129 {130 "name": "Empty history",131 "history": [],132 "expected": False133 },134 {135 "name": "Too short entries (role format)",136 "history": [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hi"}],137 "expected": False138 },139 {140 "name": "Too short entries (user/assistant format)",141 "history": [{"user": "Hi", "assistant": "Hi"}],142 "expected": False143 },144 {145 "name": "Meaningful conversation (role format)",146 "history": [{"role": "user", "content": "What is machine learning?"}, {"role": "assistant", "content": "Machine learning is a subset of artificial intelligence..."}],147 "expected": True148 },149 {150 "name": "Meaningful conversation (user/assistant format)",151 "history": [{"user": "Explain neural networks", "assistant": "Neural networks are computing systems inspired by biological neural networks..."}],152 "expected": True153 },154 {155 "name": "Mixed meaningful and short entries",156 "history": [157 {"user": "Hi", "assistant": "Hello"},158 {"user": "What is deep learning?", "assistant": "Deep learning is a subset of machine learning that uses neural networks with multiple layers..."}159 ],160 "expected": True161 },162 ]163 164 passed_tests = 0165 total_tests = len(test_cases)166 167 for i, case in enumerate(test_cases, 1):168 print(f"\n🔍 Test {i}: {case['name']}")169 170 result = has_meaningful_conversation_history(case["history"])171 172 print(f" 📝 Result: {result} (expected: {case['expected']})")173 174 if result == case["expected"]:175 print(" ✅ PASS - Behavior consistent")176 passed_tests += 1177 else:178 print(" ❌ FAIL - Behavior inconsistent")179 180 print(f"\n📊 History Analysis Tests: {passed_tests}/{total_tests} passed")181 return passed_tests == total_tests182 183 except Exception as e:184 print(f"❌ History analysis regression test error: {e}")185 return False186 187def test_utility_functions_consistency():188 """Test utility functions consistency"""189 190 print("\n🛠️ Regression Testing: Utility Functions")191 print("=" * 60)192 193 try:194 from search_optimizer import format_search_context195 196 # Test format_search_context with various inputs197 test_cases = [198 {199 "name": "Empty results list",200 "results": [],201 "expected_empty": True202 },203 {204 "name": "Single result",205 "results": [{"source": "Brave", "title": "Test Title", "body": "Test body content"}],206 "expected_contains": ["[Brave]", "Test Title", "Test body content"]207 },208 {209 "name": "Multiple results",210 "results": [211 {"source": "Brave", "title": "Title 1", "body": "Body 1"},212 {"source": "DuckDuckGo", "title": "Title 2", "body": "Body 2"}213 ],214 "expected_contains": ["[Brave]", "[DuckDuckGo]", "Title 1", "Title 2"]215 },216 {217 "name": "Results with missing fields",218 "results": [{"title": "Only Title"}, {"source": "Only Source"}],219 "expected_contains": ["Only Title", "Only Source"]220 },221 {222 "name": "Very long body content (truncation test)",223 "results": [{"source": "Test", "title": "Long Content", "body": "x" * 2000}],224 "expected_contains": ["[Test]", "Long Content"],225 "expected_truncated": True226 }227 ]228 229 passed_tests = 0230 total_tests = len(test_cases)231 232 for i, case in enumerate(test_cases, 1):233 print(f"\n🔍 Test {i}: {case['name']}")234 235 result = format_search_context(case["results"])236 237 # Check if result is empty as expected238 if case.get("expected_empty", False):239 if not result:240 print(" ✅ PASS - Empty result as expected")241 passed_tests += 1242 else:243 print(" ❌ FAIL - Expected empty result")244 continue245 246 # Check expected content247 contains_all = True248 for expected_content in case.get("expected_contains", []):249 if expected_content not in result:250 contains_all = False251 print(f" 🔸 Missing expected content: {expected_content}")252 253 # Check truncation if expected254 if case.get("expected_truncated", False):255 if len(result) < 2000: # Should be truncated from original 2000+ chars256 print(" 📏 Content appropriately truncated")257 else:258 print(" ⚠️ Content may not be truncated as expected")259 260 if contains_all:261 print(" ✅ PASS - Content formatting consistent")262 passed_tests += 1263 else:264 print(" ❌ FAIL - Content formatting issue")265 266 print(f"\n📊 Utility Function Tests: {passed_tests}/{total_tests} passed")267 return passed_tests == total_tests268 269 except Exception as e:270 print(f"❌ Utility function regression test error: {e}")271 return False272 273def test_error_handling_consistency():274 """Test that error handling behaves consistently"""275 276 print("\n🛡️ Regression Testing: Error Handling")277 print("=" * 60)278 279 try:280 from search_optimizer import should_perform_search, has_meaningful_conversation_history, format_search_context281 282 print("🔍 Testing graceful error handling...")283 284 # Test functions with malformed inputs285 error_tests_passed = 0286 total_error_tests = 0287 288 # Test should_perform_search with malformed history289 print("\n 📝 Testing should_perform_search with malformed history")290 total_error_tests += 1291 try:292 result = should_perform_search("test prompt", [{"malformed": "entry"}])293 if isinstance(result, dict) and "should_search" in result:294 print(" ✅ Handled malformed history gracefully")295 error_tests_passed += 1296 else:297 print(" ❌ Unexpected result format")298 except Exception as e:299 print(f" ❌ Unexpected exception: {e}")300 301 # Test has_meaningful_conversation_history with malformed data302 print("\n 📝 Testing has_meaningful_conversation_history with malformed data")303 total_error_tests += 1304 try:305 result = has_meaningful_conversation_history([{"invalid": "format"}, "not_a_dict"])306 if isinstance(result, bool):307 print(" ✅ Handled malformed data gracefully")308 error_tests_passed += 1309 else:310 print(" ❌ Unexpected result type")311 except Exception as e:312 print(f" ❌ Unexpected exception: {e}")313 314 # Test format_search_context with malformed results315 print("\n 📝 Testing format_search_context with malformed results")316 total_error_tests += 1317 try:318 result = format_search_context([{"missing_keys": True}, None, "not_a_dict"])319 if isinstance(result, str):320 print(" ✅ Handled malformed results gracefully")321 error_tests_passed += 1322 else:323 print(" ❌ Unexpected result type")324 except Exception as e:325 print(f" ❌ Unexpected exception: {e}")326 327 print(f"\n📊 Error Handling Tests: {error_tests_passed}/{total_error_tests} passed")328 return error_tests_passed == total_error_tests329 330 except Exception as e:331 print(f"❌ Error handling regression test error: {e}")332 return False333 334def main():335 """Run regression validation suite"""336 337 print("🔄 Search Optimizer Regression Validation Suite")338 print("=" * 70)339 print("Validating behavioral consistency after refactoring...")340 print()341 342 tests_passed = 0343 total_tests = 4344 345 # Run regression tests346 if test_search_decision_consistency():347 tests_passed += 1348 349 if test_conversation_history_consistency():350 tests_passed += 1351 352 if test_utility_functions_consistency():353 tests_passed += 1354 355 if test_error_handling_consistency():356 tests_passed += 1357 358 # Summary359 print("\n" + "=" * 70)360 print("📋 REGRESSION VALIDATION SUMMARY")361 print("=" * 70)362 363 if tests_passed == total_tests:364 print(f"✅ ALL REGRESSION TESTS PASSED ({tests_passed}/{total_tests})")365 print("🎉 Behavioral consistency maintained after refactoring!")366 print("🚀 The refactored code behaves exactly as expected!")367 return True368 else:369 print(f"❌ SOME REGRESSION TESTS FAILED ({tests_passed}/{total_tests})")370 print("⚠️ Behavioral changes detected - review required")371 return False372 373if __name__ == "__main__":374 success = main()375 sys.exit(0 if success else 1)