Sentient-Field/sgp-tribe3
0
1"""2SGP-Tribe3 — Main API Application3==================================4Multimodal brain encoding API with SGP 9-node parcellation.5Supports video+audio, audio-only, and text-only inputs via TRIBE v2.6 7Endpoints:8 GET / - Service info9 GET /health - Model load status10 POST /warmup - Trigger model loading11 POST /predict - Run inference on video file (video + audio encoding)12 POST /predict_text - Run inference on text input (text-only encoding)13 POST /predict_audio - Run inference on audio file (audio-only encoding)14 GET /nodes - SGP node definitions15 GET /tracts - White matter tract definitions16 GET /results - All stored stimulus results17 GET /coactivation_matrix - Cross-stimulus co-activation matrix18 19Reference: Harvard MLSysBook - Machine Learning Systems20https://github.com/harvard-edge/cs249r_book21"""22 23# CRITICAL: Set CPU-only mode BEFORE any torch imports24import os25os.environ['CUDA_VISIBLE_DEVICES'] = ''26os.environ['TRANSFORMERS_DEVICE'] = 'cpu'27 28import sys29import warnings30import threading31import traceback32import tempfile33import subprocess34import uuid35import math36import json37import os38import numpy as np39import pandas as pd40 41# CRITICAL: Patch torch.cuda BEFORE any ML libraries are imported42# This must be at the very top to prevent CUDA lazy initialization43_original_cuda = sys.modules.get('torch.cuda')44import torch45 46class _CPUOnlyCUDA:47 """Dummy CUDA module that always reports CPU-only mode."""48 49 @staticmethod50 def is_available():51 return False52 53 @staticmethod54 def device_count():55 return 056 57 @staticmethod58 def current_device():59 return 060 61 @staticmethod62 def device(idx=0):63 # Return a device with type 'cuda' but mapped to CPU internally64 # This allows transformers to check device.type without crashing65 d = torch.device('cpu')66 # Patch the type to appear as cuda (trick the library)67 object.__setattr__(d, 'type', 'cuda')68 return d69 70 @staticmethod71 def set_device(idx):72 pass73 74 @staticmethod75 def synchronize(device=None):76 pass77 78 @staticmethod79 def empty_cache():80 pass81 82 @staticmethod83 def memory_allocated(device=None):84 return 085 86 @staticmethod87 def memory_reserved(device=None):88 return 089 90 @staticmethod91 def reset_peak_memory_stats(device=None):92 pass93 94 # Prevent any actual CUDA operations95 def __getattr__(self, name):96 return lambda *args, **kwargs: None97 98 def __repr__(self):99 return "<CPU-only CUDA mock>"100 101# Replace torch.cuda completely102sys.modules['torch.cuda'] = _CPUOnlyCUDA()103 104# Force the torch.cuda module to be "initialized" before any code runs105import torch106 107# Most importantly: patch _lazy_init to be a no-op108# This is the function that throws the assertion error when CUDA is not compiled109try:110 # Try to patch at the C level111 torch._C._lazy_init = lambda: None112except:113 pass114 115# Patch cuda module's lazy init116import torch.cuda117if hasattr(torch.cuda, '_lazy_init'):118 torch.cuda._lazy_init = lambda: None119 120# Prevent the assertion error by making is_initialized return True121torch.cuda.is_initialized = lambda: True122torch.cuda._is_initialized = lambda: True123torch.cuda._initialized = lambda: True124 125# The key: patch _is_compiled to say YES it was compiled126# This is checked in the lazy init127if hasattr(torch, '_C'):128 torch._C._is_compiled = lambda: True129 if hasattr(torch._C, '_CudaBase__is_compiled'):130 torch._C._CudaBase__is_compiled = lambda: True131 132print("[SGP-Tribe3] Patched torch._lazy_init extensively", flush=True)133 134warnings.filterwarnings("ignore")135 136# CRITICAL: Patch transformers at the VERY TOP before importing TRIBE137# This must happen before tribev2 is imported138# PATCHING AT HIGHEST PRIORITY - MUST WORK139try:140 import transformers.modeling_utils141 import torch142 import torch.nn as nn143 144 # CRITICAL: Replace .to() on torch.nn.Module FIRST145 # This is the base class that everything inherits from146 def noop_to(self, *args, **kwargs):147 return self148 149 nn.Module.to = noop_to150 151 # Save original __init__ FIRST152 orig_init = transformers.modeling_utils.PreTrainedModel.__init__153 154 # Replace .to() completely on ALL PreTrainedModel classes155 def patched_to(self, *args, **kwargs):156 return self # No-op - don't move model anywhere157 158 def patched_init(self, *args, **kwargs):159 import torch160 if 'device' not in kwargs or kwargs['device'] is None:161 kwargs['device'] = torch.device('cpu')162 elif isinstance(kwargs['device'], str) and kwargs['device'].startswith('cuda'):163 kwargs['device'] = torch.device('cpu')164 return orig_init(self, *args, **kwargs)165 166 transformers.modeling_utils.PreTrainedModel.__init__ = patched_init167 transformers.modeling_utils.PreTrainedModel.to = patched_to168 169 print("[SGP-Tribe3] Patched nn.Module.to and transformers PreTrainedModel", flush=True)170except Exception as e:171 print(f"[SGP-Tribe3] Early patch error (non-fatal): {e}", flush=True)172 173from flask import Flask, request, jsonify174from sgp_parcellation import get_parcellator, SGP_NODE_DEFINITIONS, SGP_TRACT_DEFINITIONS175 176app = Flask(__name__)177 178# ─── Global model state ───────────────────────────────────────────────────────179_model = None180_model_loaded = False181_model_loading = False182_model_error = None183_model_lock = threading.Lock()184 185# ─── Configuration ────────────────────────────────────────────────────────────186HF_TOKEN = os.environ.get("HF_TOKEN", "")187CKPT = os.environ.get("TRIBE_CKPT", "facebook/tribev2")188MAX_VIDEO_DURATION = int(os.environ.get("MAX_VIDEO_DURATION", "120"))189MAX_AUDIO_DURATION = int(os.environ.get("MAX_AUDIO_DURATION", "120"))190CACHE_DIR = os.environ.get("SGP_CACHE_DIR", "/tmp/sgp_atlas")191 192os.environ.setdefault("HF_HUB_CACHE", "/tmp/hf_hub_cache")193os.environ.setdefault("WHISPER_CACHE_DIR", "/tmp/whisper_cache")194os.environ.setdefault("HF_HUB_DISABLE_TELEMETRY", "1")195os.makedirs(os.environ["HF_HUB_CACHE"], exist_ok=True)196os.makedirs(os.environ["WHISPER_CACHE_DIR"], exist_ok=True)197 198# ─── Result storage (in-memory for now; extend to HF dataset for persistence) ───199_stimulus_results = {}200 201# ─── Metrics tracking (MLOps best practice) ──────────────────────────────────202_metrics = {203 "start_time": None,204 "total_predictions": 0,205 "predictions_by_modality": {"video": 0, "audio": 0, "text": 0},206 "inference_times": [],207}208 209 210# ─── Model loading ────────────────────────────────────────────────────────────211 212def _load_model():213 global _model, _model_loaded, _model_loading, _model_error, _metrics214 215 with _model_lock:216 if _model_loaded or _model_loading:217 return218 _model_loading = True219 _model_error = None220 221 try:222 print("[SGP-Tribe3] Starting model load...", flush=True)223 _metrics["start_time"] = pd.Timestamp.now().isoformat()224 225 if HF_TOKEN:226 os.environ["HUGGING_FACE_HUB_TOKEN"] = HF_TOKEN227 os.environ["HF_TOKEN"] = HF_TOKEN228 try:229 from huggingface_hub import login230 login(token=HF_TOKEN, add_to_git_credential=False)231 print(f"[SGP-Tribe3] HF login OK", flush=True)232 except Exception as e:233 print(f"[SGP-Tribe3] HF login warning: {e}", flush=True)234 else:235 print("[SGP-Tribe3] WARNING: No HF_TOKEN set — LLaMA encoder may fail", flush=True)236 237 import torch238 print(f"[SGP-Tribe3] PyTorch {torch.__version__}", flush=True)239 240 # CRITICAL: Patch torch.cuda._lazy_init to not throw assertion error241 # The error happens in _lazy_init checking if torch was compiled with CUDA242 import torch.cuda243 if hasattr(torch.cuda, '_lazy_init'):244 _orig_lazy_init = torch.cuda._lazy_init245 def _safe_lazy_init():246 try:247 return _orig_lazy_init()248 except AssertionError:249 # Swallow the "Torch not compiled with CUDA enabled" error250 pass251 torch.cuda._lazy_init = _safe_lazy_init252 print("[SGP-Tribe3] Patched torch.cuda._lazy_init to be safe", flush=True)253 254 # Force CPU mode via environment255 os.environ['CUDA_VISIBLE_DEVICES'] = ''256 257 # Patch neuralset/transformers AFTER torch is imported but BEFORE model loads258 try:259 # Import first260 import neuralset.extractors.base261 # Patch the device property on all extractors to return CPU262 neuralset.extractors.base.BaseExtractor.device = property(lambda self: 'cpu')263 print("[SGP-Tribe3] Patched BaseExtractor.device to CPU", flush=True)264 265 # Patch ALL extractor subclasses266 from neuralset.extractors import audio, video, text267 for module in [audio, video, text]:268 for name in dir(module):269 cls = getattr(module, name, None)270 if cls and isinstance(cls, type) and hasattr(cls, 'device'):271 try:272 cls.device = property(lambda self: 'cpu')273 except:274 pass275 print("[SGP-Tribe3] Patched all extractor device properties", flush=True)276 except Exception as e:277 print(f"[SGP-Tribe3] Extractor patch warning: {e}", flush=True)278 279 # Also patch transformers' PreTrainedModel.to() method and __init__280 try:281 import transformers.modeling_utils282 283 # Patch PreTrainedModel.__init__ to default to cpu284 # Note: Don't use orig_init here - use the one from top of file285 orig_init = transformers.modeling_utils.PreTrainedModel.__init__286 287 def patched_init(self, *args, **kwargs):288 # Force device to cpu in kwargs289 import torch290 if 'device' not in kwargs or kwargs['device'] is None:291 kwargs['device'] = torch.device('cpu')292 elif isinstance(kwargs['device'], str) and kwargs['device'].startswith('cuda'):293 kwargs['device'] = torch.device('cpu')294 return orig_init(self, *args, **kwargs)295 296 # Also patch torch.nn.Module._apply at the base level297 import torch.nn as nn298 orig_apply = nn.Module._apply299 300 def cpu_apply(self, fn):301 # This intercepts _apply which is called by .to()302 def wrapped_fn(t):303 return t # Skip the conversion - keep on CPU304 return orig_apply(self, wrapped_fn)305 306 nn.Module._apply = cpu_apply307 308 # Also patch PreTrainedModel.to - use the top-level patch we already defined309 # Don't re-patch - just make sure it's using our no-op version310 311 # Also patch the device property to always return cpu312 try:313 import torch314 # Get the original device property315 orig_device = transformers.modeling_utils.PreTrainedModel.device316 317 def patched_device(self):318 return torch.device('cpu')319 320 # Replace the property321 transformers.modeling_utils.PreTrainedModel.device = property(patched_device)322 except Exception as e:323 print(f"[SGP-Tribe3] device property patch warning: {e}", flush=True)324 325 # Also patch all subclasses of PreTrainedModel326 import torch327 for cls_name in dir(transformers.modeling_utils):328 cls = getattr(transformers.modeling_utils, cls_name, None)329 if cls and isinstance(cls, type) and issubclass(cls, transformers.modeling_utils.PreTrainedModel):330 try:331 cls.device = property(lambda self: torch.device('cpu'))332 except:333 pass334 335 transformers.modeling_utils.PreTrainedModel.__init__ = patched_init336 print("[SGP-Tribe3] Patched transformers PreTrainedModel __init__ and device", flush=True)337 except Exception as e:338 print(f"[SGP-Tribe3] transformers patch warning: {e}", flush=True)339 340 # Load TRIBE v2 model341 from tribev2 import TribeModel342 print("[SGP-Tribe3] Loading TribeModel...", flush=True)343 model = TribeModel.from_pretrained(CKPT, device='cpu')344 print("[SGP-Tribe3] TribeModel loaded!", flush=True)345 346 # Pre-warm the parcellator (downloads Schaefer atlas if needed)347 print("[SGP-Tribe3] Initializing SGP parcellator...", flush=True)348 parcellator = get_parcellator(CACHE_DIR)349 _ = parcellator.get_vertex_map()350 print("[SGP-Tribe3] Parcellator ready!", flush=True)351 352 with _model_lock:353 _model = model354 _model_loaded = True355 _model_loading = False356 print("[SGP-Tribe3] READY", flush=True)357 358 except Exception as e:359 err = traceback.format_exc()360 print(f"[SGP-Tribe3] LOAD ERROR:\n{err}", flush=True)361 with _model_lock:362 _model_loading = False363 _model_error = str(e)364 365 366# ─── Video/Audio preprocessing ────────────────────────────────────────────────367 368def _get_video_duration(video_path: str) -> float:369 """Get video duration in seconds using ffprobe."""370 import json371 cmd = ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", video_path]372 result = subprocess.run(cmd, capture_output=True, text=True)373 if result.returncode == 0:374 try:375 data = json.loads(result.stdout)376 return float(data.get("format", {}).get("duration", 0))377 except:378 pass379 return 0.0380 381 382def _preprocess_video(video_path: str, max_duration: int = MAX_VIDEO_DURATION) -> str:383 """384 Trim video to max_duration and normalize to TRIBE v2 expected format.385 Returns path to processed video file.386 """387 actual_duration = _get_video_duration(video_path)388 clip_duration = min(max_duration, actual_duration) if actual_duration > 0 else max_duration389 390 output_path = video_path.replace(".mp4", "_processed.mp4")391 392 cmd = [393 "ffmpeg", "-y",394 "-i", video_path,395 "-t", str(clip_duration),396 "-c:v", "libx264", "-preset", "fast",397 "-c:a", "aac", "-ar", "16000", "-ac", "1",398 "-vf", "scale=320:240",399 output_path400 ]401 402 result = subprocess.run(cmd, capture_output=True, text=True)403 if result.returncode != 0:404 raise ValueError(f"ffmpeg preprocessing failed: {result.stderr}")405 406 return output_path407 408 409def _get_audio_duration(audio_path: str) -> float:410 """Get audio duration in seconds using ffprobe."""411 import json412 cmd = ["ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "json", audio_path]413 result = subprocess.run(cmd, capture_output=True, text=True)414 if result.returncode == 0:415 try:416 data = json.loads(result.stdout)417 return float(data.get("format", {}).get("duration", 0))418 except:419 pass420 return 0.0421 422 423def _preprocess_audio(audio_path: str, max_duration: int = MAX_AUDIO_DURATION) -> str:424 """425 Convert audio to wav format and normalize for TRIBE v2.426 Returns path to processed audio file.427 """428 actual_duration = _get_audio_duration(audio_path)429 clip_duration = min(max_duration, actual_duration) if actual_duration > 0 else max_duration430 431 output_path = audio_path.replace(audio_path.split(".")[-1], "wav")432 if output_path == audio_path:433 output_path = audio_path.rsplit(".", 1)[0] + "_processed.wav"434 435 cmd = [436 "ffmpeg", "-y",437 "-i", audio_path,438 "-t", str(clip_duration),439 "-ar", "16000",440 "-ac", "1",441 "-acodec", "pcm_s16le",442 output_path443 ]444 445 result = subprocess.run(cmd, capture_output=True, text=True)446 if result.returncode != 0:447 raise ValueError(f"ffmpeg audio preprocessing failed: {result.stderr}")448 449 return output_path450 451 452# ─── Core inference ──────────────────────────────────────────────────────────453 454def _run_inference_from_events(events_df: pd.DataFrame) -> dict:455 """456 Run TRIBE v2 inference from an events DataFrame and return SGP parcellation.457 This is the core function used by all three modality endpoints.458 """459 import time460 start_time = time.time()461 462 try:463 # Run TRIBE v2 prediction464 # Note: standardize_events is called inside get_loaders, so we need to ensure465 # our events have the right schema BEFORE calling predict466 preds, segments = _model.predict(events=events_df, verbose=False)467 468 # Convert to numpy469 if hasattr(preds, "numpy"):470 pred_array = preds.numpy()471 else:472 pred_array = np.array(preds)473 474 if pred_array.ndim == 1:475 pred_array = pred_array.reshape(1, -1)476 477 inference_time = time.time() - start_time478 print(f"[SGP-Tribe3] Prediction shape: {pred_array.shape}, time: {inference_time:.1f}s", flush=True)479 480 # Apply SGP parcellation481 parcellator = get_parcellator(CACHE_DIR)482 result = parcellator.parcellate(pred_array)483 484 # Add activation timeline (mean activation per timestep)485 result["activation_timeline"] = [486 round(float(np.abs(pred_array[t]).mean()), 4)487 for t in range(pred_array.shape[0])488 ]489 490 # Add inference metadata491 result["inference_time_seconds"] = round(inference_time, 2)492 result["n_segments"] = pred_array.shape[0]493 result["n_vertices"] = pred_array.shape[1]494 495 # Update metrics496 _metrics["total_predictions"] += 1497 _metrics["inference_times"].append(inference_time)498 if len(_metrics["inference_times"]) > 100:499 _metrics["inference_times"] = _metrics["inference_times"][-100:]500 501 return result502 503 except Exception as e:504 raise RuntimeError(f"TRIBE v2 inference failed: {e}")505 506 507def _run_video_inference(video_path: str) -> dict:508 """509 Run inference on video file (video + audio modalities).510 Uses TRIBE v2's get_audio_and_text_events with audio_only=True.511 """512 from tribev2.demo_utils import get_audio_and_text_events513 514 processed_path = _preprocess_video(video_path)515 actual_duration = _get_video_duration(processed_path)516 clip_duration = int(actual_duration) if actual_duration > 0 else MAX_VIDEO_DURATION517 518 try:519 # Create initial video event with ALL required columns for TRIBE v2 schema520 event = pd.DataFrame([{521 "type": "Video",522 "filepath": processed_path,523 "start": 0.0,524 "timeline": "default",525 "subject": "default",526 "duration": clip_duration,527 "offset": 0.0,528 "frequency": 1.0,529 "extra": {}530 }])531 532 # Use TRIBE v2 pipeline: extracts audio, chunks, but SKIPS whisperx533 events_df = get_audio_and_text_events(event, audio_only=True)534 535 # FIX: Ensure every single row has timeline and other required fields536 # Replace any missing/None values with defaults537 if "timeline" not in events_df.columns:538 events_df["timeline"] = "default"539 events_df["timeline"] = events_df["timeline"].fillna("default")540 541 if "subject" not in events_df.columns:542 events_df["subject"] = "default"543 events_df["subject"] = events_df["subject"].fillna("default")544 545 if "duration" not in events_df.columns:546 events_df["duration"] = MAX_VIDEO_DURATION547 events_df["duration"] = events_df["duration"].fillna(MAX_VIDEO_DURATION)548 549 if "offset" not in events_df.columns:550 events_df["offset"] = 0.0551 events_df["offset"] = events_df["offset"].fillna(0.0)552 553 if "frequency" not in events_df.columns:554 events_df["frequency"] = 1.0555 events_df["frequency"] = events_df["frequency"].fillna(1.0)556 557 if "extra" not in events_df.columns:558 events_df["extra"] = {}559 events_df["extra"] = events_df["extra"].apply(lambda x: x if x is not None else {})560 561 event_types = events_df['type'].unique().tolist()562 print(f"[SGP-Tribe3] Video inference: {len(events_df)} events, types: {event_types}", flush=True)563 564 _metrics["predictions_by_modality"]["video"] += 1565 return _run_inference_from_events(events_df)566 567 finally:568 if os.path.exists(processed_path) and processed_path != video_path:569 os.remove(processed_path)570 571 572def _run_audio_inference(audio_path: str) -> dict:573 """574 Run inference on audio file (audio-only modality).575 Uses TRIBE v2's get_audio_and_text_events with audio_only=True.576 """577 from tribev2.demo_utils import get_audio_and_text_events578 579 processed_path = _preprocess_audio(audio_path)580 actual_duration = _get_audio_duration(processed_path)581 clip_duration = int(actual_duration) if actual_duration > 0 else MAX_AUDIO_DURATION582 583 try:584 # Create initial audio event with ALL required columns585 event = pd.DataFrame([{586 "type": "Audio",587 "filepath": processed_path,588 "start": 0.0,589 "timeline": "default",590 "subject": "default",591 "duration": clip_duration,592 "offset": 0.0,593 "frequency": 1.0,594 "extra": {}595 }])596 597 # Use TRIBE v2 pipeline with audio_only=True598 events_df = get_audio_and_text_events(event, audio_only=True)599 600 # FIX: Ensure every single row has timeline and other required fields601 if "timeline" not in events_df.columns:602 events_df["timeline"] = "default"603 events_df["timeline"] = events_df["timeline"].fillna("default")604 605 if "subject" not in events_df.columns:606 events_df["subject"] = "default"607 events_df["subject"] = events_df["subject"].fillna("default")608 609 if "duration" not in events_df.columns:610 events_df["duration"] = MAX_AUDIO_DURATION611 events_df["duration"] = events_df["duration"].fillna(MAX_AUDIO_DURATION)612 613 if "offset" not in events_df.columns:614 events_df["offset"] = 0.0615 events_df["offset"] = events_df["offset"].fillna(0.0)616 617 if "frequency" not in events_df.columns:618 events_df["frequency"] = 1.0619 events_df["frequency"] = events_df["frequency"].fillna(1.0)620 621 if "extra" not in events_df.columns:622 events_df["extra"] = {}623 events_df["extra"] = events_df["extra"].apply(lambda x: x if x is not None else {})624 625 event_types = events_df['type'].unique().tolist()626 print(f"[SGP-Tribe3] Audio inference: {len(events_df)} events, types: {event_types}", flush=True)627 628 _metrics["predictions_by_modality"]["audio"] += 1629 return _run_inference_from_events(events_df)630 631 finally:632 if os.path.exists(processed_path) and processed_path != audio_path:633 os.remove(processed_path)634 635 636def _run_text_inference(text: str) -> dict:637 """638 Run inference on text input (text-only modality).639 Creates Word events manually with accumulating context.640 641 CRITICAL: Patches all neuralset extractors to use CPU before TRIBE v2 predict().642 This prevents the CUDA assertion error that occurs when audio/video extractors643 try to move to GPU during text-only inference.644 """645 words = text.split()646 if not words:647 raise ValueError("Empty text provided")648 649 word_events = []650 context = ""651 for i, word in enumerate(words):652 context = f"{context} {word}" if context else word653 word_events.append({654 "type": "Word",655 "start": 0.0,656 "duration": 1.0,657 "text": word,658 "context": context,659 "timeline": "default",660 "subject": "default",661 "sequence_id": 0,662 "sentence": text,663 "language": "english",664 "offset": 0.0,665 "frequency": 1.0,666 "filepath": None,667 "extra": {}668 })669 670 events_df = pd.DataFrame(word_events)671 672 print(f"[SGP-Tribe3] Text inference: {len(words)} words", flush=True)673 674 # CRITICAL: Patch ALL extractors to use CPU BEFORE calling predict()675 # This prevents Wav2Vec-BERT and other audio/video extractors from676 # trying to move to CUDA (which fails on CPU-only PyTorch build)677 try:678 from neuralset.extractors import base, audio, video, text679 680 # Patch BaseExtractor681 base.BaseExtractor.device = property(lambda self: 'cpu')682 base.BaseExtractor._device = 'cpu'683 684 # Patch audio extractors685 for name in dir(audio):686 cls = getattr(audio, name, None)687 if cls and isinstance(cls, type) and hasattr(cls, 'device'):688 cls.device = property(lambda self: 'cpu')689 690 # Patch video extractors 691 for name in dir(video):692 cls = getattr(video, name, None)693 if cls and isinstance(cls, type) and hasattr(cls, 'device'):694 cls.device = property(lambda self: 'cpu')695 696 # Patch text extractors697 for name in dir(text):698 cls = getattr(text, name, None)699 if cls and isinstance(cls, type) and hasattr(cls, 'device'):700 cls.device = property(lambda self: 'cpu')701 702 print("[SGP-Tribe3] Patched all extractors to CPU for text inference", flush=True)703 except Exception as e:704 print(f"[SGP-Tribe3] Extractor patch warning: {e}", flush=True)705 706 _metrics["predictions_by_modality"]["text"] += 1707 result = _run_inference_from_events(events_df)708 result["text_length"] = len(text)709 result["word_count"] = len(words)710 711 return result712 713 714# ─── Routes ───────────────────────────────────────────────────────────────────715 716@app.route("/", methods=["GET"])717def index():718 return jsonify({719 "service": "SGP-Tribe3",720 "version": "1.1.0",721 "description": "Sentient Generative Principal — Brain Encoding Calibration System",722 "status": "ok",723 "modality_support": {724 "video": "Video + audio encoding (V-JEPA2 + DINOv2 + Wav2Vec-BERT)",725 "audio": "Audio-only encoding (Wav2Vec-BERT)",726 "text": "Text-only encoding (LLaMA 3.2 embeddings)"727 },728 "endpoints": {729 "GET /health": "Model load status",730 "POST /warmup": "Trigger model loading",731 "POST /predict": "Run inference on video file (multipart/form-data, field: video)",732 "POST /predict_text": "Run inference on text (form field: text)",733 "POST /predict_audio": "Run inference on audio file (field: audio)",734 "GET /nodes": "SGP node definitions",735 "GET /tracts": "White matter tract definitions",736 "GET /results": "All stored stimulus results",737 "GET /coactivation_matrix": "Cross-stimulus co-activation matrix",738 "GET /metrics": "Service metrics and monitoring",739 }740 })741 742 743@app.route("/health", methods=["GET"])744def health():745 return jsonify({746 "status": "ready" if _model_loaded else ("loading" if _model_loading else "offline"),747 "model_loaded": _model_loaded,748 "model_loading": _model_loading,749 "error": _model_error,750 "n_stored_results": len(_stimulus_results),751 })752 753 754@app.route("/warmup", methods=["POST"])755def warmup():756 if not _model_loaded and not _model_loading:757 threading.Thread(target=_load_model, daemon=True).start()758 return jsonify({759 "status": "warming_up",760 "model_loaded": _model_loaded,761 "model_loading": _model_loading,762 })763 764 765@app.route("/predict", methods=["POST"])766def predict():767 """Run inference on video file (video + audio modalities)."""768 if not _model_loaded:769 return jsonify({770 "error": "Model not loaded. POST to /warmup first.",771 "model_loading": _model_loading,772 "load_error": _model_error,773 }), 503774 775 if "video" not in request.files:776 return jsonify({"error": "No video file provided. Use multipart/form-data with 'video' field."}), 400777 778 video_file = request.files["video"]779 if video_file.filename == "":780 return jsonify({"error": "Empty filename"}), 400781 782 stimulus_id = request.form.get("stimulus_id", str(uuid.uuid4()))783 stimulus_label = request.form.get("label", "unlabeled")784 target_node = request.form.get("target_node", "unknown")785 786 suffix = os.path.splitext(video_file.filename)[1] or ".mp4"787 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:788 video_file.save(tmp.name)789 tmp_path = tmp.name790 791 try:792 print(f"[SGP-Tribe3] Video inference: stimulus_id={stimulus_id}, label={stimulus_label}", flush=True)793 result = _run_video_inference(tmp_path)794 795 result["stimulus_id"] = stimulus_id796 result["label"] = stimulus_label797 result["target_node"] = target_node798 result["modality"] = "video"799 800 _stimulus_results[stimulus_id] = result801 return jsonify({"status": "ok", "result": result})802 803 except Exception as e:804 err = traceback.format_exc()805 print(f"[SGP-Tribe3] Video inference error:\n{err}", flush=True)806 return jsonify({"error": str(e), "trace": err}), 500807 808 finally:809 if os.path.exists(tmp_path):810 os.remove(tmp_path)811 812 813@app.route("/predict_text", methods=["POST"])814def predict_text():815 """Run inference on plain text input (text-only modality)."""816 if not _model_loaded:817 return jsonify({818 "error": "Model not loaded. POST to /warmup first.",819 "model_loading": _model_loading,820 "load_error": _model_error,821 }), 503822 823 text = request.form.get("text", "").strip()824 if not text:825 return jsonify({"error": "No text provided. Use form field 'text'."}), 400826 827 stimulus_id = request.form.get("stimulus_id", str(uuid.uuid4()))828 stimulus_label = request.form.get("label", "text_stimulus")829 target_node = request.form.get("target_node", "unknown")830 831 try:832 print(f"[SGP-Tribe3] Text inference: stimulus_id={stimulus_id}, label={stimulus_label}", flush=True)833 result = _run_text_inference(text)834 835 result["stimulus_id"] = stimulus_id836 result["label"] = stimulus_label837 result["target_node"] = target_node838 result["modality"] = "text"839 840 _stimulus_results[stimulus_id] = result841 return jsonify({"status": "ok", "result": result})842 843 except Exception as e:844 err = traceback.format_exc()845 print(f"[SGP-Tribe3] Text inference error:\n{err}", flush=True)846 return jsonify({"error": str(e), "trace": err}), 500847 848 849@app.route("/predict_audio", methods=["POST"])850def predict_audio():851 """Run inference on audio file (audio-only modality)."""852 if not _model_loaded:853 return jsonify({854 "error": "Model not loaded. POST to /warmup first.",855 "model_loading": _model_loading,856 "load_error": _model_error,857 }), 503858 859 if "audio" not in request.files:860 return jsonify({"error": "No audio file. Use multipart/form-data with 'audio' field."}), 400861 862 audio_file = request.files["audio"]863 if audio_file.filename == "":864 return jsonify({"error": "Empty filename"}), 400865 866 stimulus_id = request.form.get("stimulus_id", str(uuid.uuid4()))867 stimulus_label = request.form.get("label", "audio_stimulus")868 target_node = request.form.get("target_node", "unknown")869 870 suffix = os.path.splitext(audio_file.filename)[1] or ".wav"871 with tempfile.NamedTemporaryFile(suffix=suffix, delete=False) as tmp:872 audio_file.save(tmp.name)873 tmp_path = tmp.name874 875 try:876 print(f"[SGP-Tribe3] Audio inference: stimulus_id={stimulus_id}, label={stimulus_label}", flush=True)877 result = _run_audio_inference(tmp_path)878 879 result["stimulus_id"] = stimulus_id880 result["label"] = stimulus_label881 result["target_node"] = target_node882 result["modality"] = "audio"883 884 _stimulus_results[stimulus_id] = result885 return jsonify({"status": "ok", "result": result})886 887 except Exception as e:888 err = traceback.format_exc()889 print(f"[SGP-Tribe3] Audio inference error:\n{err}", flush=True)890 return jsonify({"error": str(e), "trace": err}), 500891 892 finally:893 if os.path.exists(tmp_path):894 os.remove(tmp_path)895 896 897@app.route("/nodes", methods=["GET"])898def nodes():899 return jsonify({900 "sgp_nodes": SGP_NODE_DEFINITIONS,901 "count": len(SGP_NODE_DEFINITIONS),902 })903 904 905@app.route("/tracts", methods=["GET"])906def tracts():907 return jsonify({908 "white_matter_tracts": SGP_TRACT_DEFINITIONS,909 "count": len(SGP_TRACT_DEFINITIONS),910 })911 912 913@app.route("/results", methods=["GET"])914def results():915 return jsonify({916 "n_results": len(_stimulus_results),917 "results": _stimulus_results,918 })919 920 921@app.route("/coactivation_matrix", methods=["GET"])922def coactivation_matrix():923 """Compute the co-activation matrix across all stored stimulus results."""924 if len(_stimulus_results) < 2:925 return jsonify({926 "error": "Need at least 2 stimulus results to compute co-activation matrix.",927 "n_results": len(_stimulus_results),928 }), 400929 930 node_ids = list(SGP_NODE_DEFINITIONS.keys())931 activation_matrix = []932 stimulus_labels = []933 modalities = []934 935 for sid, res in _stimulus_results.items():936 row = [res["sgp_nodes"].get(nid, 0.0) for nid in node_ids]937 activation_matrix.append(row)938 stimulus_labels.append(res.get("label", sid))939 modalities.append(res.get("modality", "unknown"))940 941 A = np.array(activation_matrix)942 943 if A.shape[0] > 1:944 corr_matrix = np.corrcoef(A.T)945 else:946 corr_matrix = np.eye(len(node_ids))947 948 mean_activation = A.mean(axis=0)949 950 return jsonify({951 "node_ids": node_ids,952 "n_stimuli": len(_stimulus_results),953 "stimulus_labels": stimulus_labels,954 "modalities": modalities,955 "coactivation_matrix": corr_matrix.round(4).tolist(),956 "mean_activation_per_node": dict(zip(node_ids, mean_activation.round(4).tolist())),957 "interpretation": "coactivation_matrix[i][j] = Pearson correlation of node_i and node_j activation across stimuli. Use as Resonance Graph edge weights.",958 })959 960 961@app.route("/metrics", methods=["GET"])962def metrics():963 """Service metrics following MLOps best practices (MLSysBook Ch 13)."""964 mean_inference_time = 0.0965 if _metrics["inference_times"]:966 mean_inference_time = round(sum(_metrics["inference_times"]) / len(_metrics["inference_times"]), 2)967 968 uptime_seconds = None969 if _metrics["start_time"]:970 uptime_seconds = (pd.Timestamp.now() - pd.Timestamp(_metrics["start_time"])).total_seconds()971 972 return jsonify({973 "service_uptime_seconds": uptime_seconds,974 "total_predictions": _metrics["total_predictions"],975 "predictions_by_modality": _metrics["predictions_by_modality"],976 "mean_inference_time_seconds": mean_inference_time,977 "n_stored_results": len(_stimulus_results),978 "model_loaded": _model_loaded,979 })980 981 982# ─── Entry point ──────────────────────────────────────────────────────────────983 984if __name__ == "__main__":985 threading.Thread(target=_load_model, daemon=True).start()986 port = int(os.environ.get("PORT", 7860))987 app.run(host="0.0.0.0", port=port, debug=False)988