nifty-coder/stemsplit-backend
0
1"""2Unit tests for the ProviderManager class.3 4This module tests the provider management functionality including registration,5discovery, health checking, and provider selection logic.6"""7 8import pytest9import asyncio10from unittest.mock import AsyncMock, Mock, patch11from datetime import datetime, timedelta12 13from voice_control.provider_manager import ProviderManager14from voice_control.models import (15 ProviderConfig, ProviderType, TranscriptionResult, ProviderStatus, QuotaStatus, QuotaType16)17from voice_control.exceptions import (18 NoProvidersAvailableError, ProviderUnavailableError, QuotaExceededError,19 UnsupportedFormatError, UnsupportedLanguageError20)21from voice_control.interfaces import STTProvider, RateLimiterInterface22 23 24class MockSTTProvider(STTProvider):25 """Mock STT provider for testing."""26 27 def __init__(self, config: ProviderConfig):28 super().__init__(config)29 self.health_status = True30 self.quota_status = {}31 self.transcription_result = TranscriptionResult(32 text="test transcription",33 confidence=0.95,34 provider=config.name,35 processing_time=1.0,36 audio_duration=2.0,37 language="en-US"38 )39 self.should_fail = False40 self.failure_message = "Mock failure"41 42 async def transcribe_audio(self, audio_data: bytes, format: str, language: str = "en-US", **kwargs):43 if self.should_fail:44 raise Exception(self.failure_message)45 return self.transcription_result46 47 async def transcribe_streaming(self, audio_stream, format: str, language: str = "en-US", **kwargs):48 if self.should_fail:49 raise Exception(self.failure_message)50 async for chunk in audio_stream:51 yield self.transcription_result52 53 async def check_health(self) -> bool:54 return self.health_status55 56 async def get_quota_status(self):57 return self.quota_status58 59 def supports_format(self, format: str) -> bool:60 return format in self.config.supported_formats61 62 def supports_language(self, language: str) -> bool:63 return language in self.config.supported_languages64 65 async def estimate_cost(self, audio_duration: float) -> float:66 return audio_duration * self.config.cost_per_minute / 60.067 68 69class MockRateLimiter(RateLimiterInterface):70 """Mock rate limiter for testing."""71 72 def __init__(self):73 self.quota_exceeded = False74 self.quota_status = QuotaStatus(75 provider="test",76 quota_type=QuotaType.REQUESTS_PER_MINUTE,77 current_usage=5,78 limit=10,79 remaining=5,80 reset_time=datetime.utcnow() + timedelta(minutes=1),81 percentage_used=0.582 )83 self.provider_configs = {}84 self.should_switch_providers = {}85 self.provider_usage = {} # Track usage per provider86 87 async def configure_provider(self, provider_config):88 """Configure a provider for testing."""89 self.provider_configs[provider_config.name] = provider_config90 # Initialize usage tracking91 self.provider_usage[provider_config.name] = 0.092 93 async def should_switch_provider(self, provider: str, audio_duration: float = 0.0, threshold: float = 0.9):94 """Mock should switch provider method."""95 # Check if explicitly set to switch96 if provider in self.should_switch_providers:97 return self.should_switch_providers[provider]98 99 # Check if provider is approaching limits based on usage100 if provider in self.provider_configs and provider in self.provider_usage:101 config = self.provider_configs[provider]102 current_usage = self.provider_usage[provider]103 104 # Check audio minutes per day limit105 if "audio_minutes_per_day" in config.free_tier_limits:106 limit = config.free_tier_limits["audio_minutes_per_day"] * 60.0 # Convert to seconds107 projected_usage = current_usage + audio_duration108 usage_percentage = projected_usage / limit109 110 if usage_percentage >= threshold:111 return True112 113 return False114 115 async def check_quota(self, provider: str, audio_duration: float):116 if self.quota_exceeded:117 self.quota_status.current_usage = self.quota_status.limit + 1118 return self.quota_status119 120 # Create provider-specific quota status121 if provider in self.provider_configs and provider in self.provider_usage:122 config = self.provider_configs[provider]123 current_usage = self.provider_usage[provider]124 125 # Check audio minutes per day limit126 if "audio_minutes_per_day" in config.free_tier_limits:127 limit = config.free_tier_limits["audio_minutes_per_day"] * 60.0 # Convert to seconds128 projected_usage = current_usage + audio_duration129 130 return QuotaStatus(131 provider=provider,132 quota_type=QuotaType.AUDIO_MINUTES_PER_DAY,133 current_usage=projected_usage / 60.0, # Convert back to minutes134 limit=config.free_tier_limits["audio_minutes_per_day"],135 remaining=max(0, (limit - projected_usage) / 60.0),136 reset_time=datetime.utcnow() + timedelta(days=1),137 percentage_used=0.0 # Will be calculated in __post_init__138 )139 140 return self.quota_status141 142 async def consume_quota(self, provider: str, audio_duration: float, request_count: int = 1):143 if self.quota_exceeded:144 return False145 146 # Track usage per provider147 if provider not in self.provider_usage:148 self.provider_usage[provider] = 0.0149 150 self.provider_usage[provider] += audio_duration151 return True152 153 async def get_usage_stats(self, provider: str, time_window: str):154 pass155 156 async def reset_quota(self, provider: str, quota_type: str):157 if provider in self.provider_usage:158 self.provider_usage[provider] = 0.0159 160 161@pytest.fixture162def provider_manager():163 """Create a ProviderManager instance for testing."""164 return ProviderManager()165 166 167@pytest.fixture168def provider_manager_with_rate_limiter():169 """Create a ProviderManager with rate limiter for testing."""170 rate_limiter = MockRateLimiter()171 return ProviderManager(rate_limiter=rate_limiter), rate_limiter172 173 174@pytest.fixture175def free_provider_config():176 """Create a free tier provider configuration."""177 return ProviderConfig(178 name="free_provider",179 provider_type=ProviderType.WEB_SPEECH_API,180 enabled=True,181 priority=1,182 free_tier_limits={"requests_per_day": 1000},183 rate_limits={"requests_per_minute": 10},184 supported_formats=["webm", "wav"],185 supported_languages=["en-US", "en-GB"],186 cost_per_minute=0.0, # Free187 api_credentials={"api_key": "free_key"}188 )189 190 191@pytest.fixture192def paid_provider_config():193 """Create a paid provider configuration."""194 return ProviderConfig(195 name="paid_provider",196 provider_type=ProviderType.DEEPGRAM,197 enabled=True,198 priority=2,199 free_tier_limits={},200 rate_limits={"requests_per_minute": 100},201 supported_formats=["webm", "wav", "mp3"],202 supported_languages=["en-US", "es-ES"],203 cost_per_minute=0.05, # Paid204 api_credentials={"api_key": "paid_key"}205 )206 207 208class TestProviderManager:209 """Test cases for ProviderManager class."""210 211 @pytest.mark.asyncio212 async def test_provider_registration(self, provider_manager, free_provider_config):213 """Test registering a provider."""214 provider = MockSTTProvider(free_provider_config)215 216 await provider_manager.register_provider(provider)217 218 # Check provider is registered219 assert "free_provider" in provider_manager._providers220 assert provider_manager._providers["free_provider"] == provider221 222 # Check status is initialized223 status = await provider_manager.get_provider_status()224 assert "free_provider" in status225 assert status["free_provider"].name == "free_provider"226 assert status["free_provider"].available is True227 228 @pytest.mark.asyncio229 async def test_duplicate_provider_registration_fails(self, provider_manager, free_provider_config):230 """Test that registering a provider with duplicate name fails."""231 provider1 = MockSTTProvider(free_provider_config)232 provider2 = MockSTTProvider(free_provider_config)233 234 await provider_manager.register_provider(provider1)235 236 with pytest.raises(ValueError, match="already registered"):237 await provider_manager.register_provider(provider2)238 239 @pytest.mark.asyncio240 async def test_provider_unregistration(self, provider_manager, free_provider_config):241 """Test unregistering a provider."""242 provider = MockSTTProvider(free_provider_config)243 244 await provider_manager.register_provider(provider)245 await provider_manager.unregister_provider("free_provider")246 247 # Check provider is removed248 assert "free_provider" not in provider_manager._providers249 250 status = await provider_manager.get_provider_status()251 assert "free_provider" not in status252 253 @pytest.mark.asyncio254 async def test_unregister_nonexistent_provider_fails(self, provider_manager):255 """Test that unregistering a non-existent provider fails."""256 with pytest.raises(KeyError, match="not registered"):257 await provider_manager.unregister_provider("nonexistent")258 259 @pytest.mark.asyncio260 async def test_provider_config_update(self, provider_manager, free_provider_config):261 """Test updating provider configuration."""262 provider = MockSTTProvider(free_provider_config)263 await provider_manager.register_provider(provider)264 265 # Update config266 new_config = free_provider_config267 new_config.priority = 5268 269 result = await provider_manager.update_provider_config("free_provider", new_config)270 271 assert result is True272 assert provider_manager._providers["free_provider"].config.priority == 5273 274 @pytest.mark.asyncio275 async def test_update_nonexistent_provider_config_fails(self, provider_manager, free_provider_config):276 """Test that updating config for non-existent provider fails."""277 result = await provider_manager.update_provider_config("nonexistent", free_provider_config)278 assert result is False279 280 @pytest.mark.asyncio281 async def test_free_tier_prioritization(self, provider_manager, free_provider_config, paid_provider_config):282 """Test that free tier providers are prioritized over paid ones."""283 # Create providers with same priority but different costs284 free_provider_config.priority = 2285 paid_provider_config.priority = 1 # Lower priority number but paid286 287 free_provider = MockSTTProvider(free_provider_config)288 paid_provider = MockSTTProvider(paid_provider_config)289 290 await provider_manager.register_provider(paid_provider)291 await provider_manager.register_provider(free_provider)292 293 # Get ordered providers294 ordered = await provider_manager._get_ordered_providers("webm", "en-US")295 296 # Free provider should come first despite higher priority number297 assert ordered[0] == "free_provider"298 assert ordered[1] == "paid_provider"299 300 @pytest.mark.asyncio301 async def test_preferred_provider_gets_highest_priority(self, provider_manager, free_provider_config, paid_provider_config):302 """Test that preferred provider gets highest priority regardless of other factors."""303 free_provider = MockSTTProvider(free_provider_config)304 paid_provider = MockSTTProvider(paid_provider_config)305 306 await provider_manager.register_provider(free_provider)307 await provider_manager.register_provider(paid_provider)308 309 # Get ordered providers with preferred provider310 ordered = await provider_manager._get_ordered_providers("webm", "en-US", preferred_provider="paid_provider")311 312 # Paid provider should come first because it's preferred313 assert ordered[0] == "paid_provider"314 assert ordered[1] == "free_provider"315 316 @pytest.mark.asyncio317 async def test_format_filtering(self, provider_manager, free_provider_config):318 """Test that providers are filtered by supported formats."""319 provider = MockSTTProvider(free_provider_config)320 await provider_manager.register_provider(provider)321 322 # Request supported format323 ordered = await provider_manager._get_ordered_providers("webm", "en-US")324 assert "free_provider" in ordered325 326 # Request unsupported format327 ordered = await provider_manager._get_ordered_providers("mp3", "en-US")328 assert "free_provider" not in ordered329 330 @pytest.mark.asyncio331 async def test_language_filtering(self, provider_manager, free_provider_config):332 """Test that providers are filtered by supported languages."""333 provider = MockSTTProvider(free_provider_config)334 await provider_manager.register_provider(provider)335 336 # Request supported language337 ordered = await provider_manager._get_ordered_providers("webm", "en-US")338 assert "free_provider" in ordered339 340 # Request unsupported language341 ordered = await provider_manager._get_ordered_providers("webm", "fr-FR")342 assert "free_provider" not in ordered343 344 @pytest.mark.asyncio345 async def test_disabled_provider_filtering(self, provider_manager, free_provider_config):346 """Test that disabled providers are filtered out."""347 free_provider_config.enabled = False348 provider = MockSTTProvider(free_provider_config)349 await provider_manager.register_provider(provider)350 351 ordered = await provider_manager._get_ordered_providers("webm", "en-US")352 assert "free_provider" not in ordered353 354 @pytest.mark.asyncio355 async def test_successful_transcription(self, provider_manager, free_provider_config, sample_audio_data):356 """Test successful audio transcription."""357 provider = MockSTTProvider(free_provider_config)358 await provider_manager.register_provider(provider)359 360 result = await provider_manager.transcribe_audio(sample_audio_data, "webm", "en-US")361 362 assert result.text == "test transcription"363 assert result.provider == "free_provider"364 assert result.confidence == 0.95365 366 @pytest.mark.asyncio367 async def test_transcription_with_no_providers(self, provider_manager, sample_audio_data):368 """Test transcription fails when no providers are available."""369 with pytest.raises(NoProvidersAvailableError):370 await provider_manager.transcribe_audio(sample_audio_data, "webm", "en-US")371 372 @pytest.mark.asyncio373 async def test_transcription_fallback_on_failure(self, provider_manager, free_provider_config, paid_provider_config, sample_audio_data):374 """Test that transcription falls back to next provider on failure."""375 # Set up providers with different priorities376 free_provider_config.priority = 1377 paid_provider_config.priority = 2378 379 free_provider = MockSTTProvider(free_provider_config)380 paid_provider = MockSTTProvider(paid_provider_config)381 382 # Make free provider fail383 free_provider.should_fail = True384 free_provider.failure_message = "Free provider failed"385 386 await provider_manager.register_provider(free_provider)387 await provider_manager.register_provider(paid_provider)388 389 result = await provider_manager.transcribe_audio(sample_audio_data, "webm", "en-US")390 391 # Should get result from paid provider (fallback)392 assert result.provider == "paid_provider"393 394 @pytest.mark.asyncio395 async def test_circuit_breaker_opens_after_failures(self, provider_manager, free_provider_config, sample_audio_data):396 """Test that circuit breaker opens after consecutive failures."""397 provider = MockSTTProvider(free_provider_config)398 provider.should_fail = True399 400 await provider_manager.register_provider(provider)401 402 # Cause multiple failures to trigger circuit breaker403 for _ in range(6): # Threshold is 5404 try:405 await provider_manager.transcribe_audio(sample_audio_data, "webm", "en-US")406 except:407 pass408 409 # Check circuit breaker is open410 status = await provider_manager.get_provider_status()411 assert status["free_provider"].circuit_breaker_state == "OPEN"412 assert status["free_provider"].consecutive_failures >= 5413 414 @pytest.mark.asyncio415 async def test_circuit_breaker_reset(self, provider_manager, free_provider_config):416 """Test manual circuit breaker reset."""417 provider = MockSTTProvider(free_provider_config)418 await provider_manager.register_provider(provider)419 420 # Manually set circuit breaker to open421 provider_manager._provider_status["free_provider"].circuit_breaker_state = "OPEN"422 provider_manager._provider_status["free_provider"].consecutive_failures = 10423 424 # Reset circuit breaker425 result = await provider_manager.reset_provider_circuit_breaker("free_provider")426 427 assert result is True428 status = await provider_manager.get_provider_status()429 assert status["free_provider"].circuit_breaker_state == "CLOSED"430 assert status["free_provider"].consecutive_failures == 0431 432 @pytest.mark.asyncio433 async def test_rate_limiter_integration(self, free_provider_config, sample_audio_data):434 """Test integration with rate limiter."""435 manager, rate_limiter = provider_manager_with_rate_limiter()436 provider = MockSTTProvider(free_provider_config)437 438 await manager.register_provider(provider)439 440 # Test successful transcription with quota available441 result = await manager.transcribe_audio(sample_audio_data, "webm", "en-US")442 assert result.provider == "free_provider"443 444 # Test quota exceeded scenario445 rate_limiter.quota_exceeded = True446 447 with pytest.raises(NoProvidersAvailableError):448 await manager.transcribe_audio(sample_audio_data, "webm", "en-US")449 450 @pytest.mark.asyncio451 async def test_provider_metrics_tracking(self, provider_manager, free_provider_config, sample_audio_data):452 """Test that provider metrics are tracked correctly."""453 provider = MockSTTProvider(free_provider_config)454 await provider_manager.register_provider(provider)455 456 # Perform successful transcription457 await provider_manager.transcribe_audio(sample_audio_data, "webm", "en-US")458 459 # Check metrics460 metrics = await provider_manager.get_provider_metrics()461 assert "free_provider" in metrics462 463 provider_metrics = metrics["free_provider"]464 assert provider_metrics["total_requests"] == 1465 assert provider_metrics["successful_requests"] == 1466 assert provider_metrics["failed_requests"] == 0467 468 @pytest.mark.asyncio469 async def test_health_check_updates_status(self, provider_manager, free_provider_config):470 """Test that health checks update provider status."""471 provider = MockSTTProvider(free_provider_config)472 provider.health_status = False # Set unhealthy473 474 await provider_manager.register_provider(provider)475 476 # Perform health check477 await provider_manager._check_provider_health("free_provider", provider)478 479 # Check status updated480 status = await provider_manager.get_provider_status()481 assert status["free_provider"].available is False482 483 @pytest.mark.asyncio484 async def test_streaming_transcription(self, provider_manager, free_provider_config):485 """Test streaming transcription functionality."""486 provider = MockSTTProvider(free_provider_config)487 await provider_manager.register_provider(provider)488 489 # Create mock audio stream490 async def audio_stream():491 yield b"chunk1"492 yield b"chunk2"493 494 results = []495 async for result in provider_manager.transcribe_streaming(audio_stream(), "webm", "en-US"):496 results.append(result)497 498 assert len(results) == 2499 assert all(result.provider == "free_provider" for result in results)500 501 @pytest.mark.asyncio502 async def test_automatic_provider_switching_integration(self, free_provider_config, paid_provider_config, sample_audio_data):503 """Test automatic provider switching when quota is approached."""504 manager, rate_limiter = provider_manager_with_rate_limiter()505 506 # Configure providers with different quota limits507 free_provider_config.free_tier_limits = {"audio_minutes_per_day": 5} # Small limit508 paid_provider_config.free_tier_limits = {"audio_minutes_per_day": 60} # Large limit509 510 free_provider = MockSTTProvider(free_provider_config)511 paid_provider = MockSTTProvider(paid_provider_config)512 513 await manager.register_provider(free_provider)514 await manager.register_provider(paid_provider)515 516 # Configure rate limiter with the same providers517 await rate_limiter.configure_provider(free_provider_config)518 await rate_limiter.configure_provider(paid_provider_config)519 520 # Set up mock to simulate free provider approaching limits521 rate_limiter.should_switch_providers["free_provider"] = True522 rate_limiter.should_switch_providers["paid_provider"] = False523 524 # Next transcription should automatically switch to paid provider525 result = await manager.transcribe_audio(sample_audio_data, "webm", "en-US")526 527 # Should use paid provider due to free provider approaching limits528 assert result.provider == "paid_provider"529 530 @pytest.mark.asyncio531 async def test_get_alternative_provider(self, provider_manager, free_provider_config, paid_provider_config, sample_audio_data):532 """Test getting alternative provider when current one is approaching limits."""533 free_provider = MockSTTProvider(free_provider_config)534 paid_provider = MockSTTProvider(paid_provider_config)535 536 await provider_manager.register_provider(free_provider)537 await provider_manager.register_provider(paid_provider)538 539 # Get alternative to free provider540 alternative = await provider_manager.get_alternative_provider(541 "free_provider", "webm", "en-US", 30.0542 )543 544 assert alternative == "paid_provider"545 546 # Get alternative to paid provider547 alternative = await provider_manager.get_alternative_provider(548 "paid_provider", "webm", "en-US", 30.0549 )550 551 assert alternative == "free_provider"552 553 @pytest.mark.asyncio554 async def test_recommend_provider_switch(self, free_provider_config, paid_provider_config):555 """Test provider switch recommendation functionality."""556 manager, rate_limiter = provider_manager_with_rate_limiter()557 558 # Configure providers559 free_provider_config.free_tier_limits = {"audio_minutes_per_day": 10}560 paid_provider_config.free_tier_limits = {"audio_minutes_per_day": 60}561 562 free_provider = MockSTTProvider(free_provider_config)563 paid_provider = MockSTTProvider(paid_provider_config)564 565 await manager.register_provider(free_provider)566 await manager.register_provider(paid_provider)567 568 await rate_limiter.configure_provider(free_provider_config)569 await rate_limiter.configure_provider(paid_provider_config)570 571 # Set up mock to recommend switching for free provider572 rate_limiter.should_switch_providers["free_provider"] = True573 rate_limiter.should_switch_providers["paid_provider"] = False574 575 # Get switch recommendation576 recommendation = await manager.recommend_provider_switch(577 "free_provider", "webm", "en-US", 120.0 # 2 more minutes578 )579 580 assert recommendation is not None581 assert recommendation["should_switch"] is True582 assert recommendation["alternative_provider"] == "paid_provider"583 assert "approaching limits" in recommendation["reason"].lower() or "quota" in recommendation["reason"].lower()584 585 @pytest.mark.asyncio586 async def test_no_switch_recommendation_when_quota_available(self, free_provider_config, paid_provider_config):587 """Test that no switch is recommended when quota is available."""588 manager, rate_limiter = provider_manager_with_rate_limiter()589 590 free_provider = MockSTTProvider(free_provider_config)591 paid_provider = MockSTTProvider(paid_provider_config)592 593 await manager.register_provider(free_provider)594 await manager.register_provider(paid_provider)595 596 await rate_limiter.configure_provider(free_provider_config)597 await rate_limiter.configure_provider(paid_provider_config)598 599 # Don't consume much quota600 await rate_limiter.consume_quota("free_provider", 60.0, 1) # 1 minute601 602 # Get switch recommendation603 recommendation = await manager.recommend_provider_switch(604 "free_provider", "webm", "en-US", 60.0 # 1 more minute605 )606 607 assert recommendation is None608 609 @pytest.mark.asyncio610 async def test_provider_ordering_considers_quota_status(self, free_provider_config, paid_provider_config):611 """Test that provider ordering considers quota status."""612 manager, rate_limiter = provider_manager_with_rate_limiter()613 614 # Configure providers with same priority but different quota usage615 free_provider_config.priority = 1616 paid_provider_config.priority = 1617 free_provider_config.free_tier_limits = {"audio_minutes_per_day": 10}618 paid_provider_config.free_tier_limits = {"audio_minutes_per_day": 60}619 620 free_provider = MockSTTProvider(free_provider_config)621 paid_provider = MockSTTProvider(paid_provider_config)622 623 await manager.register_provider(free_provider)624 await manager.register_provider(paid_provider)625 626 await rate_limiter.configure_provider(free_provider_config)627 await rate_limiter.configure_provider(paid_provider_config)628 629 # Consume most of free provider's quota630 await rate_limiter.consume_quota("free_provider", 540.0, 1) # 9 minutes out of 10631 632 # Get ordered providers633 ordered = await manager._get_ordered_providers("webm", "en-US", audio_duration=120.0)634 635 # Paid provider should come first due to better quota availability636 # (even though free provider normally has priority)637 assert ordered[0] == "paid_provider"638 assert ordered[1] == "free_provider"639 640 @pytest.mark.asyncio641 async def test_shutdown_cleanup(self, provider_manager, free_provider_config):642 """Test that shutdown properly cleans up resources."""643 provider = MockSTTProvider(free_provider_config)644 await provider_manager.register_provider(provider)645 646 # Shutdown647 await provider_manager.shutdown()648 649 # Check cleanup650 assert len(provider_manager._providers) == 0651 assert len(provider_manager._provider_status) == 0652 assert len(provider_manager._provider_metrics) == 0653 654 655@pytest.fixture656def sample_audio_data():657 """Create sample audio data for testing."""658 return b"fake_audio_data_for_testing"659 660 661def provider_manager_with_rate_limiter():662 """Helper function to create provider manager with rate limiter."""663 rate_limiter = MockRateLimiter()664 manager = ProviderManager(rate_limiter=rate_limiter)665 return manager, rate_limiter666 667 668@pytest.mark.integration669class TestProviderManagerCacheIntegration:670 """Test cache integration with ProviderManager (Task 5.2)."""671 672 @pytest.fixture673 def cache_manager(self):674 """Create a mock cache manager for testing."""675 from voice_control.cache_manager import CacheManager676 return CacheManager(max_size_mb=10, default_ttl=3600, enable_stats=True)677 678 @pytest.fixture679 def provider_manager_with_cache(self, cache_manager):680 """Create provider manager with cache integration."""681 return ProviderManager(cache_manager=cache_manager)682 683 @pytest.fixture684 def sample_transcription_result(self):685 """Create a sample transcription result."""686 return TranscriptionResult(687 text="cached transcription result",688 confidence=0.95,689 provider="cache_test_provider",690 processing_time=1.5,691 audio_duration=3.0,692 language="en-US",693 alternatives=["alternative result"]694 )695 696 @pytest.mark.asyncio697 async def test_cache_hit_avoids_provider_calls(self, provider_manager_with_cache, free_provider_config, sample_audio_data, sample_transcription_result):698 """699 Test that cache hits avoid calling STT providers (Requirement 4.3).700 701 **Validates: Requirements 4.3**702 """703 # Start cache manager704 await provider_manager_with_cache._cache_manager.start()705 706 try:707 # Pre-populate cache708 fingerprint = await provider_manager_with_cache._cache_manager.generate_audio_fingerprint(sample_audio_data)709 await provider_manager_with_cache._cache_manager.cache_transcription(fingerprint, sample_transcription_result)710 711 # Create a provider that should NOT be called712 provider = MockSTTProvider(free_provider_config)713 provider.should_fail = True # If called, it will fail714 provider.failure_message = "Provider should not be called due to cache hit"715 716 await provider_manager_with_cache.register_provider(provider)717 718 # Transcribe audio - should get cached result without calling provider719 result = await provider_manager_with_cache.transcribe_audio(720 sample_audio_data, 721 format="webm",722 language="en-US"723 )724 725 # Should get cached result, not provider result726 assert result.text == sample_transcription_result.text727 assert result.provider == sample_transcription_result.provider728 assert result.confidence == sample_transcription_result.confidence729 730 # Verify cache statistics show hit731 cache_stats = await provider_manager_with_cache.get_cache_stats()732 assert cache_stats is not None733 assert cache_stats["cache_hits"] >= 1734 735 finally:736 await provider_manager_with_cache._cache_manager.stop()737 738 @pytest.mark.asyncio739 async def test_cache_miss_calls_provider_and_caches_result(self, provider_manager_with_cache, free_provider_config, sample_audio_data):740 """741 Test that cache misses call providers and cache the result.742 743 **Validates: Requirements 4.1, 4.3**744 """745 # Start cache manager746 await provider_manager_with_cache._cache_manager.start()747 748 try:749 # Create provider with specific result750 provider_result = TranscriptionResult(751 text="provider transcription result",752 confidence=0.90,753 provider=free_provider_config.name,754 processing_time=2.0,755 audio_duration=3.0,756 language="en-US"757 )758 759 provider = MockSTTProvider(free_provider_config)760 provider.transcription_result = provider_result761 762 await provider_manager_with_cache.register_provider(provider)763 764 # First transcription - should be cache miss and call provider765 result1 = await provider_manager_with_cache.transcribe_audio(766 sample_audio_data,767 format="webm",768 language="en-US"769 )770 771 # Should get provider result772 assert result1.text == provider_result.text773 assert result1.provider == provider_result.provider774 775 # Now make provider fail to ensure second call uses cache776 provider.should_fail = True777 provider.failure_message = "Provider should not be called on second request"778 779 # Second transcription - should be cache hit780 result2 = await provider_manager_with_cache.transcribe_audio(781 sample_audio_data,782 format="webm", 783 language="en-US"784 )785 786 # Should get same result from cache787 assert result2.text == result1.text788 assert result2.provider == result1.provider789 assert result2.confidence == result1.confidence790 791 # Verify cache statistics792 cache_stats = await provider_manager_with_cache.get_cache_stats()793 assert cache_stats is not None794 assert cache_stats["cache_hits"] >= 1795 assert cache_stats["cache_misses"] >= 1796 assert cache_stats["total_requests"] >= 2797 798 finally:799 await provider_manager_with_cache._cache_manager.stop()800 801 @pytest.mark.asyncio802 async def test_different_audio_produces_different_cache_entries(self, provider_manager_with_cache, free_provider_config):803 """804 Test that different audio produces different cache entries.805 806 **Validates: Requirements 4.5**807 """808 # Start cache manager809 await provider_manager_with_cache._cache_manager.start()810 811 try:812 # Create provider813 provider = MockSTTProvider(free_provider_config)814 await provider_manager_with_cache.register_provider(provider)815 816 # Different audio samples817 audio1 = b"first_audio_sample" * 100818 audio2 = b"second_audio_sample" * 100819 820 # Transcribe first audio821 result1 = await provider_manager_with_cache.transcribe_audio(822 audio1,823 format="webm",824 language="en-US"825 )826 827 # Transcribe second audio828 result2 = await provider_manager_with_cache.transcribe_audio(829 audio2,830 format="webm",831 language="en-US"832 )833 834 # Both should be cache misses (different audio)835 cache_stats = await provider_manager_with_cache.get_cache_stats()836 assert cache_stats is not None837 assert cache_stats["cache_misses"] >= 2838 assert cache_stats["entry_count"] >= 2839 840 finally:841 await provider_manager_with_cache._cache_manager.stop()842 843 @pytest.mark.asyncio844 async def test_cache_statistics_integration(self, provider_manager_with_cache, free_provider_config):845 """846 Test comprehensive cache statistics through provider manager.847 848 **Validates: Requirements 4.3**849 """850 # Start cache manager851 await provider_manager_with_cache._cache_manager.start()852 853 try:854 # Create provider855 provider = MockSTTProvider(free_provider_config)856 await provider_manager_with_cache.register_provider(provider)857 858 # Perform multiple transcriptions859 audio_samples = [860 b"audio_sample_1" * 50,861 b"audio_sample_2" * 50,862 b"audio_sample_1" * 50, # Repeat first sample for cache hit863 b"audio_sample_3" * 50,864 b"audio_sample_2" * 50, # Repeat second sample for cache hit865 ]866 867 for audio_data in audio_samples:868 await provider_manager_with_cache.transcribe_audio(869 audio_data,870 format="webm",871 language="en-US"872 )873 874 # Check cache statistics875 cache_stats = await provider_manager_with_cache.get_cache_stats()876 assert cache_stats is not None877 878 # Should have 5 total requests: 3 misses + 2 hits879 assert cache_stats["total_requests"] == 5880 assert cache_stats["cache_misses"] == 3 # First occurrence of each unique audio881 assert cache_stats["cache_hits"] == 2 # Repeated audio samples882 assert cache_stats["entry_count"] == 3 # 3 unique audio samples cached883 884 # Hit rate should be 2/5 = 0.4885 expected_hit_rate = 2.0 / 5.0886 assert abs(cache_stats["hit_rate"] - expected_hit_rate) < 0.01887 888 # Check system stats integration889 system_stats = await provider_manager_with_cache.get_system_stats()890 assert system_stats["cache"] is not None891 assert system_stats["cache"]["total_requests"] == 5892 assert system_stats["system_overview"]["cache_enabled"] is True893 894 finally:895 await provider_manager_with_cache._cache_manager.stop()896 897 @pytest.mark.asyncio898 async def test_cache_failure_does_not_break_transcription(self, provider_manager_with_cache, free_provider_config, sample_audio_data):899 """900 Test that cache failures don't break transcription functionality.901 902 **Validates: Requirements 4.3**903 """904 # Start cache manager905 await provider_manager_with_cache._cache_manager.start()906 907 try:908 # Create provider909 provider = MockSTTProvider(free_provider_config)910 await provider_manager_with_cache.register_provider(provider)911 912 # Mock cache manager to fail on operations913 original_get_cached = provider_manager_with_cache._cache_manager.get_cached_transcription914 original_cache_transcription = provider_manager_with_cache._cache_manager.cache_transcription915 916 async def failing_get_cached(*args, **kwargs):917 raise Exception("Cache lookup failed")918 919 async def failing_cache_transcription(*args, **kwargs):920 raise Exception("Cache storage failed")921 922 provider_manager_with_cache._cache_manager.get_cached_transcription = failing_get_cached923 provider_manager_with_cache._cache_manager.cache_transcription = failing_cache_transcription924 925 # Transcription should still work despite cache failures926 result = await provider_manager_with_cache.transcribe_audio(927 sample_audio_data,928 format="webm",929 language="en-US"930 )931 932 # Should get provider result933 assert result is not None934 assert result.text == provider.transcription_result.text935 936 # Restore original methods937 provider_manager_with_cache._cache_manager.get_cached_transcription = original_get_cached938 provider_manager_with_cache._cache_manager.cache_transcription = original_cache_transcription939 940 finally:941 await provider_manager_with_cache._cache_manager.stop()942 943 @pytest.mark.asyncio944 async def test_provider_manager_without_cache(self, free_provider_config, sample_audio_data):945 """946 Test that provider manager works correctly without cache manager.947 948 **Validates: Requirements 4.3**949 """950 # Create provider manager without cache951 provider_manager = ProviderManager()952 953 # Create provider954 provider = MockSTTProvider(free_provider_config)955 await provider_manager.register_provider(provider)956 957 # Transcription should work normally958 result = await provider_manager.transcribe_audio(959 sample_audio_data,960 format="webm",961 language="en-US"962 )963 964 assert result is not None965 assert result.text == provider.transcription_result.text966 967 # Cache stats should be None968 cache_stats = await provider_manager.get_cache_stats()969 assert cache_stats is None970 971 # System stats should show cache disabled972 system_stats = await provider_manager.get_system_stats()973 assert system_stats["cache"] is None974 assert system_stats["system_overview"]["cache_enabled"] is False