ahmedmiloudi/BioTechLabAI
1
1"""2Cache Manager Module3====================4Système de cache persistant intelligent pour drug discovery5 6Fonctionnalités:7- Cache SQLite persistant8- TTL (Time-To-Live) configurable9- Compression automatique10- Statistiques et monitoring11- Cleanup automatique12 13Auteur: Drug Discovery Platform14Date: 2026-01-3115"""16 17import sqlite318import pickle19import gzip20import json21from datetime import datetime, timedelta22from typing import Any, Optional, Dict, List23from pathlib import Path24import logging25from functools import wraps26import hashlib27import pandas as pd28logger = logging.getLogger(__name__)29 30 31# ============================================================================32# CACHE MANAGER CLASS33# ============================================================================34 35class CacheManager:36 """37 Gestionnaire de cache persistant avec SQLite38 39 Features:40 - Stockage persistant entre sessions41 - TTL automatique42 - Compression pour économiser espace43 - Statistiques détaillées44 - Nettoyage automatique des entrées expirées45 """46 47 def __init__(self, 48 db_path: str = "cache/drug_discovery_cache.db",49 default_ttl: int = 3600,50 max_size_mb: int = 500,51 compress: bool = True):52 """53 Initialize cache manager54 55 Args:56 db_path: Chemin vers base de données SQLite57 default_ttl: TTL par défaut en secondes (1h par défaut)58 max_size_mb: Taille max du cache en MB59 compress: Activer compression gzip60 """61 self.db_path = Path(db_path)62 self.db_path.parent.mkdir(parents=True, exist_ok=True)63 64 self.default_ttl = default_ttl65 self.max_size_mb = max_size_mb66 self.compress = compress67 68 self.conn = None69 self._connect()70 self._create_tables()71 72 logger.info(f"CacheManager initialized: {db_path}")73 74 def _connect(self):75 """Établit connexion à la base de données"""76 self.conn = sqlite3.connect(str(self.db_path), check_same_thread=False)77 self.conn.row_factory = sqlite3.Row78 79 def _create_tables(self):80 """Crée tables du cache"""81 with self.conn:82 # Table principale cache83 self.conn.execute('''84 CREATE TABLE IF NOT EXISTS cache_entries (85 key TEXT PRIMARY KEY,86 value BLOB NOT NULL,87 created_at REAL NOT NULL,88 expires_at REAL NOT NULL,89 ttl INTEGER NOT NULL,90 compressed INTEGER NOT NULL,91 data_type TEXT,92 size_bytes INTEGER,93 access_count INTEGER DEFAULT 0,94 last_accessed REAL95 )96 ''')97 98 # Table statistiques99 self.conn.execute('''100 CREATE TABLE IF NOT EXISTS cache_stats (101 id INTEGER PRIMARY KEY AUTOINCREMENT,102 timestamp REAL NOT NULL,103 total_entries INTEGER,104 total_size_mb REAL,105 hit_rate REAL,106 expired_cleaned INTEGER107 )108 ''')109 110 # Index pour performance111 self.conn.execute('''112 CREATE INDEX IF NOT EXISTS idx_expires_at 113 ON cache_entries(expires_at)114 ''')115 116 self.conn.execute('''117 CREATE INDEX IF NOT EXISTS idx_created_at 118 ON cache_entries(created_at)119 ''')120 121 # ========================================================================122 # CORE OPERATIONS123 # ========================================================================124 125 def get(self, key: str) -> Optional[Any]:126 """127 Récupère valeur du cache128 129 Args:130 key: Clé de cache131 132 Returns:133 Valeur décodée ou None si expirée/inexistante134 """135 try:136 cursor = self.conn.execute(137 'SELECT value, compressed, expires_at FROM cache_entries WHERE key = ?',138 (key,)139 )140 141 row = cursor.fetchone()142 143 if not row:144 return None145 146 # Vérifier expiration147 if datetime.now().timestamp() > row['expires_at']:148 self._delete(key)149 return None150 151 # Décoder valeur152 value_bytes = row['value']153 154 if row['compressed']:155 value_bytes = gzip.decompress(value_bytes)156 157 value = pickle.loads(value_bytes)158 159 # Mettre à jour statistiques d'accès160 self._update_access_stats(key)161 162 return value163 164 except Exception as e:165 logger.error(f"Error retrieving from cache: {str(e)}")166 return None167 168 def set(self, 169 key: str, 170 value: Any, 171 ttl: Optional[int] = None,172 data_type: Optional[str] = None):173 """174 Stocke valeur dans le cache175 176 Args:177 key: Clé de cache178 value: Valeur à cacher179 ttl: Time-to-live en secondes (None = default)180 data_type: Type de données (pour stats)181 """182 try:183 if ttl is None:184 ttl = self.default_ttl185 186 # Sérialiser187 value_bytes = pickle.dumps(value)188 original_size = len(value_bytes)189 190 # Compresser si activé et si gain > 10%191 compressed = False192 if self.compress and original_size > 1024: # > 1KB193 compressed_bytes = gzip.compress(value_bytes, compresslevel=6)194 if len(compressed_bytes) < original_size * 0.9:195 value_bytes = compressed_bytes196 compressed = True197 198 # Timestamps199 now = datetime.now().timestamp()200 expires_at = now + ttl201 202 # Inférer type si non fourni203 if data_type is None:204 data_type = type(value).__name__205 206 # Insérer/mettre à jour207 with self.conn:208 self.conn.execute('''209 INSERT OR REPLACE INTO cache_entries 210 (key, value, created_at, expires_at, ttl, compressed, data_type, size_bytes, access_count, last_accessed)211 VALUES (?, ?, ?, ?, ?, ?, ?, ?, 0, ?)212 ''', (key, value_bytes, now, expires_at, ttl, int(compressed), data_type, len(value_bytes), now))213 214 # Vérifier taille totale215 self._check_size_limit()216 217 except Exception as e:218 logger.error(f"Error storing in cache: {str(e)}")219 220 def delete(self, key: str):221 """Supprime entrée du cache"""222 self._delete(key)223 224 def _delete(self, key: str):225 """Supprime entrée (méthode interne)"""226 with self.conn:227 self.conn.execute('DELETE FROM cache_entries WHERE key = ?', (key,))228 229 def exists(self, key: str) -> bool:230 """Vérifie si clé existe et est valide"""231 return self.get(key) is not None232 233 # ========================================================================234 # BATCH OPERATIONS235 # ========================================================================236 237 def get_many(self, keys: List[str]) -> Dict[str, Any]:238 """Récupère plusieurs valeurs"""239 results = {}240 for key in keys:241 value = self.get(key)242 if value is not None:243 results[key] = value244 return results245 246 def set_many(self, items: Dict[str, Any], ttl: Optional[int] = None):247 """Stocke plusieurs valeurs"""248 for key, value in items.items():249 self.set(key, value, ttl=ttl)250 251 def delete_many(self, keys: List[str]):252 """Supprime plusieurs entrées"""253 with self.conn:254 placeholders = ','.join('?' * len(keys))255 self.conn.execute(256 f'DELETE FROM cache_entries WHERE key IN ({placeholders})',257 keys258 )259 260 # ========================================================================261 # CLEANUP & MAINTENANCE262 # ========================================================================263 264 def cleanup_expired(self) -> int:265 """266 Nettoie entrées expirées267 268 Returns:269 Nombre d'entrées supprimées270 """271 now = datetime.now().timestamp()272 273 cursor = self.conn.execute(274 'SELECT COUNT(*) as count FROM cache_entries WHERE expires_at < ?',275 (now,)276 )277 count = cursor.fetchone()['count']278 279 if count > 0:280 with self.conn:281 self.conn.execute('DELETE FROM cache_entries WHERE expires_at < ?', (now,))282 283 logger.info(f"Cleaned up {count} expired cache entries")284 285 return count286 287 def _check_size_limit(self):288 """Vérifie et applique limite de taille"""289 stats = self.get_stats()290 291 if stats['total_size_mb'] > self.max_size_mb:292 # Supprimer les entrées les plus anciennes non accédées293 to_delete = int(stats['total_entries'] * 0.1) # Supprimer 10%294 295 with self.conn:296 self.conn.execute('''297 DELETE FROM cache_entries298 WHERE key IN (299 SELECT key FROM cache_entries300 ORDER BY last_accessed ASC301 LIMIT ?302 )303 ''', (to_delete,))304 305 logger.warning(f"Cache size limit reached. Deleted {to_delete} oldest entries")306 307 def clear_all(self):308 """Vide complètement le cache"""309 with self.conn:310 self.conn.execute('DELETE FROM cache_entries')311 312 logger.info("Cache cleared completely")313 314 def vacuum(self):315 """Optimise base de données"""316 self.conn.execute('VACUUM')317 logger.info("Database vacuumed")318 319 # ========================================================================320 # STATISTICS321 # ========================================================================322 323 def _update_access_stats(self, key: str):324 """Met à jour stats d'accès"""325 now = datetime.now().timestamp()326 327 with self.conn:328 self.conn.execute('''329 UPDATE cache_entries 330 SET access_count = access_count + 1,331 last_accessed = ?332 WHERE key = ?333 ''', (now, key))334 335 def get_stats(self) -> Dict[str, Any]:336 """337 Récupère statistiques du cache338 339 Returns:340 Dict avec statistiques341 """342 # Total entries343 cursor = self.conn.execute('SELECT COUNT(*) as count FROM cache_entries')344 total_entries = cursor.fetchone()['count']345 346 # Total size347 cursor = self.conn.execute('SELECT SUM(size_bytes) as total FROM cache_entries')348 total_bytes = cursor.fetchone()['total'] or 0349 total_size_mb = total_bytes / (1024 * 1024)350 351 # Entrées expirées352 now = datetime.now().timestamp()353 cursor = self.conn.execute(354 'SELECT COUNT(*) as count FROM cache_entries WHERE expires_at < ?',355 (now,)356 )357 expired_count = cursor.fetchone()['count']358 359 # Compression stats360 cursor = self.conn.execute(361 'SELECT COUNT(*) as count FROM cache_entries WHERE compressed = 1'362 )363 compressed_count = cursor.fetchone()['count']364 365 # Types de données366 cursor = self.conn.execute('''367 SELECT data_type, COUNT(*) as count 368 FROM cache_entries 369 GROUP BY data_type370 ''')371 data_types = {row['data_type']: row['count'] for row in cursor.fetchall()}372 373 # Most accessed374 cursor = self.conn.execute('''375 SELECT key, access_count 376 FROM cache_entries 377 ORDER BY access_count DESC 378 LIMIT 5379 ''')380 top_accessed = [381 {'key': row['key'], 'count': row['access_count']}382 for row in cursor.fetchall()383 ]384 385 stats = {386 'total_entries': total_entries,387 'total_size_mb': round(total_size_mb, 2),388 'expired_entries': expired_count,389 'valid_entries': total_entries - expired_count,390 'compressed_entries': compressed_count,391 'compression_ratio': round(compressed_count / total_entries * 100, 1) if total_entries > 0 else 0,392 'data_types': data_types,393 'top_accessed': top_accessed,394 'max_size_mb': self.max_size_mb,395 'usage_percent': round(total_size_mb / self.max_size_mb * 100, 1),396 'db_path': str(self.db_path)397 }398 399 return stats400 401 def save_stats_snapshot(self):402 """Sauvegarde snapshot des statistiques"""403 stats = self.get_stats()404 405 with self.conn:406 self.conn.execute('''407 INSERT INTO cache_stats 408 (timestamp, total_entries, total_size_mb, hit_rate, expired_cleaned)409 VALUES (?, ?, ?, ?, ?)410 ''', (411 datetime.now().timestamp(),412 stats['total_entries'],413 stats['total_size_mb'],414 0.0, # Hit rate nécessite tracking séparé415 stats['expired_entries']416 ))417 418 def get_stats_history(self, hours: int = 24) -> pd.DataFrame:419 """Récupère historique des stats"""420 import pandas as pd421 422 cutoff = (datetime.now() - timedelta(hours=hours)).timestamp()423 424 cursor = self.conn.execute('''425 SELECT * FROM cache_stats 426 WHERE timestamp > ?427 ORDER BY timestamp DESC428 ''', (cutoff,))429 430 rows = cursor.fetchall()431 432 if not rows:433 return pd.DataFrame()434 435 df = pd.DataFrame([dict(row) for row in rows])436 df['timestamp'] = pd.to_datetime(df['timestamp'], unit='s')437 438 return df439 440 # ========================================================================441 # DECORATOR442 # ========================================================================443 444 def cached(self, ttl: Optional[int] = None, key_prefix: str = ""):445 """446 Decorator pour cacher résultats de fonction447 448 Usage:449 cache = CacheManager()450 451 @cache.cached(ttl=3600, key_prefix="uniprot")452 def get_protein_data(gene_name):453 # ... expensive operation454 return data455 """456 def decorator(func):457 @wraps(func)458 def wrapper(*args, **kwargs):459 # Générer clé460 key_parts = [key_prefix, func.__name__]461 key_parts.extend(str(arg) for arg in args)462 key_parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))463 464 cache_key = hashlib.md5(":".join(key_parts).encode()).hexdigest()465 466 # Essayer cache467 cached_value = self.get(cache_key)468 if cached_value is not None:469 logger.debug(f"Cache hit for {func.__name__}")470 return cached_value471 472 # Calculer et cacher473 logger.debug(f"Cache miss for {func.__name__}")474 result = func(*args, **kwargs)475 self.set(cache_key, result, ttl=ttl)476 477 return result478 479 return wrapper480 return decorator481 482 # ========================================================================483 # CONTEXT MANAGER484 # ========================================================================485 486 def __enter__(self):487 return self488 489 def __exit__(self, exc_type, exc_val, exc_tb):490 self.close()491 492 def close(self):493 """Ferme connexion"""494 if self.conn:495 self.conn.close()496 logger.info("Cache connection closed")497 498 499# ============================================================================500# UTILITY FUNCTIONS501# ============================================================================502 503def generate_cache_key(*args, **kwargs) -> str:504 """Génère clé de cache depuis arguments"""505 parts = [str(arg) for arg in args]506 parts.extend(f"{k}={v}" for k, v in sorted(kwargs.items()))507 return hashlib.md5(":".join(parts).encode()).hexdigest()508 509 510# ============================================================================511# EXAMPLE USAGE512# ============================================================================513 514if __name__ == "__main__":515 import pandas as pd516 517 # Créer cache manager518 cache = CacheManager(519 db_path="cache/test_cache.db",520 default_ttl=3600,521 max_size_mb=100522 )523 524 # Test basique525 print("Testing cache operations...")526 527 # Set528 cache.set("test_key", {"data": [1, 2, 3], "name": "test"}, ttl=60)529 530 # Get531 value = cache.get("test_key")532 print(f"Retrieved: {value}")533 534 # Stats535 stats = cache.get_stats()536 print("\nCache Statistics:")537 for key, val in stats.items():538 if key not in ['data_types', 'top_accessed']:539 print(f" {key}: {val}")540 541 # Test decorator542 @cache.cached(ttl=120, key_prefix="math")543 def expensive_calculation(n):544 import time545 time.sleep(2) # Simulate slow operation546 return n ** 2547 548 print("\nTesting cached function...")549 print("First call (slow):")550 result = expensive_calculation(10)551 print(f"Result: {result}")552 553 print("Second call (fast, from cache):")554 result = expensive_calculation(10)555 print(f"Result: {result}")556 557 # Cleanup558 cleaned = cache.cleanup_expired()559 print(f"\nCleaned {cleaned} expired entries")560 561 # Close562 cache.close()563