AI-Talent-Force/dev_caio
0
1"""2ShortSmith v2 - Pipeline Orchestrator Module3 4Main coordinator for the highlight extraction pipeline.5Manages the flow between all components:61. Video preprocessing72. Scene detection83. Frame sampling94. Audio analysis105. Visual analysis116. Person detection (optional)127. Hype scoring138. Clip extraction14"""15 16from pathlib import Path17from typing import List, Optional, Callable, Dict, Any18from dataclasses import dataclass, field19from enum import Enum20import time21import traceback22 23from utils.logger import get_logger, LogTimer24from utils.helpers import (25 get_temp_dir,26 cleanup_temp_files,27 validate_video_file,28 validate_image_file,29 VideoProcessingError,30)31from config import get_config, AppConfig, ContentDomain32from core.video_processor import VideoProcessor, VideoMetadata33from core.scene_detector import SceneDetector, Scene34from core.frame_sampler import FrameSampler, SampledFrame35from core.clip_extractor import ClipExtractor, ExtractedClip, ClipCandidate36from models.audio_analyzer import AudioAnalyzer, AudioFeatures37from models.visual_analyzer import VisualAnalyzer, VisualFeatures38from models.face_recognizer import FaceRecognizer39from models.body_recognizer import BodyRecognizer40from models.motion_detector import MotionDetector41from scoring.hype_scorer import HypeScorer, SegmentScore42from scoring.domain_presets import get_domain_preset43from scoring.viral_hooks import ViralHookDetector, HookSignal44 45logger = get_logger("pipeline.orchestrator")46 47 48class PipelineStage(Enum):49 """Pipeline processing stages."""50 INITIALIZING = "initializing"51 LOADING_VIDEO = "loading_video"52 DETECTING_SCENES = "detecting_scenes"53 EXTRACTING_AUDIO = "extracting_audio"54 ANALYZING_AUDIO = "analyzing_audio"55 SAMPLING_FRAMES = "sampling_frames"56 ANALYZING_VISUAL = "analyzing_visual"57 DETECTING_PERSON = "detecting_person"58 ANALYZING_MOTION = "analyzing_motion"59 SCORING = "scoring"60 OPTIMIZING_HOOKS = "optimizing_hooks"61 EXTRACTING_CLIPS = "extracting_clips"62 FINALIZING = "finalizing"63 COMPLETE = "complete"64 FAILED = "failed"65 66 67@dataclass68class PipelineProgress:69 """Progress information for the pipeline."""70 stage: PipelineStage71 progress: float # 0.0 to 1.072 message: str73 elapsed_time: float = 0.074 estimated_remaining: float = 0.075 76 def to_dict(self) -> Dict[str, Any]:77 return {78 "stage": self.stage.value,79 "progress": round(self.progress, 2),80 "message": self.message,81 "elapsed_time": round(self.elapsed_time, 1),82 "estimated_remaining": round(self.estimated_remaining, 1),83 }84 85 86@dataclass87class PipelineResult:88 """Result of pipeline execution."""89 success: bool90 clips: List[ExtractedClip] = field(default_factory=list)91 metadata: Optional[VideoMetadata] = None92 scores: List[SegmentScore] = field(default_factory=list)93 error_message: Optional[str] = None94 processing_time: float = 0.095 temp_dir: Optional[Path] = None96 97 # Intermediate results (for debugging)98 scenes: List[Scene] = field(default_factory=list)99 audio_features: List[AudioFeatures] = field(default_factory=list)100 visual_features: List[VisualFeatures] = field(default_factory=list)101 102 def to_dict(self) -> Dict[str, Any]:103 return {104 "success": self.success,105 "num_clips": len(self.clips),106 "clips": [c.to_dict() for c in self.clips],107 "error": self.error_message,108 "processing_time": round(self.processing_time, 1),109 "video_duration": self.metadata.duration if self.metadata else 0,110 }111 112 113class PipelineOrchestrator:114 """115 Main orchestrator for the ShortSmith highlight extraction pipeline.116 117 Coordinates all components and manages the processing flow.118 """119 120 # Stage weights for progress calculation121 STAGE_WEIGHTS = {122 PipelineStage.INITIALIZING: 0.02,123 PipelineStage.LOADING_VIDEO: 0.03,124 PipelineStage.DETECTING_SCENES: 0.05,125 PipelineStage.EXTRACTING_AUDIO: 0.05,126 PipelineStage.ANALYZING_AUDIO: 0.10,127 PipelineStage.SAMPLING_FRAMES: 0.10,128 PipelineStage.ANALYZING_VISUAL: 0.25,129 PipelineStage.DETECTING_PERSON: 0.10,130 PipelineStage.ANALYZING_MOTION: 0.05,131 PipelineStage.SCORING: 0.05,132 PipelineStage.OPTIMIZING_HOOKS: 0.05,133 PipelineStage.EXTRACTING_CLIPS: 0.10,134 PipelineStage.FINALIZING: 0.05,135 }136 137 def __init__(138 self,139 config: Optional[AppConfig] = None,140 progress_callback: Optional[Callable[[PipelineProgress], None]] = None,141 ):142 """143 Initialize pipeline orchestrator.144 145 Args:146 config: Application configuration147 progress_callback: Function to call with progress updates148 """149 self.config = config or get_config()150 self.progress_callback = progress_callback151 152 self._start_time = 0.0153 self._current_stage = PipelineStage.INITIALIZING154 self._temp_dir: Optional[Path] = None155 156 # Components (lazy loaded)157 self._video_processor: Optional[VideoProcessor] = None158 self._scene_detector: Optional[SceneDetector] = None159 self._frame_sampler: Optional[FrameSampler] = None160 self._audio_analyzer: Optional[AudioAnalyzer] = None161 self._visual_analyzer: Optional[VisualAnalyzer] = None162 self._face_recognizer: Optional[FaceRecognizer] = None163 self._body_recognizer: Optional[BodyRecognizer] = None164 self._motion_detector: Optional[MotionDetector] = None165 self._clip_extractor: Optional[ClipExtractor] = None166 self._hype_scorer: Optional[HypeScorer] = None167 self._hook_detector: Optional[ViralHookDetector] = None168 169 logger.info("PipelineOrchestrator initialized")170 171 def _update_progress(172 self,173 stage: PipelineStage,174 stage_progress: float,175 message: str,176 ) -> None:177 """Update progress and call callback."""178 self._current_stage = stage179 180 # Calculate overall progress181 completed_weight = sum(182 w for s, w in self.STAGE_WEIGHTS.items()183 if list(PipelineStage).index(s) < list(PipelineStage).index(stage)184 )185 current_weight = self.STAGE_WEIGHTS.get(stage, 0)186 overall_progress = completed_weight + (current_weight * stage_progress)187 188 elapsed = time.time() - self._start_time189 190 # Estimate remaining time191 if overall_progress > 0:192 estimated_total = elapsed / overall_progress193 estimated_remaining = max(0, estimated_total - elapsed)194 else:195 estimated_remaining = 0196 197 progress = PipelineProgress(198 stage=stage,199 progress=overall_progress,200 message=message,201 elapsed_time=elapsed,202 estimated_remaining=estimated_remaining,203 )204 205 logger.debug(f"Progress: {stage.value} - {stage_progress*100:.0f}% - {message}")206 207 if self.progress_callback:208 try:209 self.progress_callback(progress)210 except Exception as e:211 logger.warning(f"Progress callback error: {e}")212 213 def process(214 self,215 video_path: str | Path,216 num_clips: int = 3,217 clip_duration: float = 15.0,218 domain: str = "general",219 reference_image: Optional[str | Path] = None,220 custom_prompt: Optional[str] = None,221 api_key: Optional[str] = None,222 ) -> PipelineResult:223 """224 Process a video and extract highlight clips.225 226 Args:227 video_path: Path to the input video228 num_clips: Number of clips to extract229 clip_duration: Target clip duration in seconds230 domain: Content domain for scoring weights231 reference_image: Reference image for person filtering (optional)232 custom_prompt: Custom instructions for analysis (optional)233 api_key: API key for external services (optional, for future use)234 235 Returns:236 PipelineResult with extracted clips and metadata237 """238 self._start_time = time.time()239 video_path = Path(video_path)240 241 logger.info(f"Starting pipeline for: {video_path.name}")242 logger.info(f"Parameters: clips={num_clips}, duration={clip_duration}s, domain={domain}")243 244 try:245 # Initialize246 self._update_progress(PipelineStage.INITIALIZING, 0.0, "Initializing pipeline...")247 self._temp_dir = get_temp_dir("shortsmith_")248 self._initialize_components(domain, reference_image is not None)249 self._update_progress(PipelineStage.INITIALIZING, 1.0, "Pipeline initialized")250 251 # Validate input252 self._update_progress(PipelineStage.LOADING_VIDEO, 0.0, "Validating video file...")253 validation = validate_video_file(video_path)254 if not validation.is_valid:255 raise VideoProcessingError(validation.error_message)256 257 # Get video metadata258 self._update_progress(PipelineStage.LOADING_VIDEO, 0.5, "Loading video metadata...")259 metadata = self._video_processor.get_metadata(video_path)260 logger.info(f"Video: {metadata.resolution}, {metadata.duration:.1f}s, {metadata.fps:.1f}fps")261 self._update_progress(PipelineStage.LOADING_VIDEO, 1.0, "Video loaded")262 263 # Check duration limit264 if metadata.duration > self.config.processing.max_video_duration:265 raise VideoProcessingError(266 f"Video too long: {metadata.duration:.0f}s "267 f"(max: {self.config.processing.max_video_duration:.0f}s)"268 )269 270 # Scene detection271 self._update_progress(PipelineStage.DETECTING_SCENES, 0.0, "Detecting scenes...")272 scenes = self._scene_detector.detect_scenes(video_path)273 self._update_progress(PipelineStage.DETECTING_SCENES, 1.0, f"Detected {len(scenes)} scenes")274 275 # Audio extraction and analysis276 self._update_progress(PipelineStage.EXTRACTING_AUDIO, 0.0, "Extracting audio...")277 audio_path = self._temp_dir / "audio.wav"278 self._video_processor.extract_audio(video_path, audio_path)279 self._update_progress(PipelineStage.EXTRACTING_AUDIO, 1.0, "Audio extracted")280 281 self._update_progress(PipelineStage.ANALYZING_AUDIO, 0.0, "Analyzing audio...")282 audio_features = self._audio_analyzer.analyze_file(audio_path)283 audio_scores = self._audio_analyzer.compute_hype_scores(audio_features)284 self._update_progress(PipelineStage.ANALYZING_AUDIO, 1.0, f"Analyzed {len(audio_features)} segments")285 286 # Frame sampling287 self._update_progress(PipelineStage.SAMPLING_FRAMES, 0.0, "Sampling frames...")288 frames = self._frame_sampler.sample_coarse(289 video_path,290 self._temp_dir / "frames",291 metadata,292 )293 self._update_progress(PipelineStage.SAMPLING_FRAMES, 1.0, f"Sampled {len(frames)} frames")294 295 # Visual analysis (if enabled)296 visual_features = []297 custom_analysis_results = []298 if self._visual_analyzer is not None:299 self._update_progress(PipelineStage.ANALYZING_VISUAL, 0.0, "Analyzing visual content...")300 try:301 for i, frame in enumerate(frames):302 features = self._visual_analyzer.analyze_frame(303 frame.frame_path, timestamp=frame.timestamp304 )305 visual_features.append(features)306 307 # Apply custom prompt analysis if provided308 if custom_prompt:309 custom_result = self._visual_analyzer.analyze_with_custom_prompt(310 frame.frame_path,311 prompt=custom_prompt,312 timestamp=frame.timestamp,313 )314 custom_analysis_results.append(custom_result)315 316 self._update_progress(317 PipelineStage.ANALYZING_VISUAL,318 (i + 1) / len(frames),319 f"Analyzing frame {i+1}/{len(frames)}"320 )321 322 # Boost scores based on custom prompt matches323 if custom_analysis_results:324 self._apply_custom_prompt_boost(visual_features, custom_analysis_results)325 326 except Exception as e:327 logger.warning(f"Visual analysis failed, continuing without: {e}")328 self._update_progress(PipelineStage.ANALYZING_VISUAL, 1.0, "Visual analysis complete")329 330 # Person detection (if reference provided)331 person_scores = []332 if reference_image and self._face_recognizer:333 self._update_progress(PipelineStage.DETECTING_PERSON, 0.0, "Detecting target person...")334 try:335 # Register reference336 ref_validation = validate_image_file(reference_image)337 if ref_validation.is_valid:338 self._face_recognizer.register_reference(reference_image)339 if self._body_recognizer:340 self._body_recognizer.register_reference(reference_image)341 342 # Detect in frames343 for i, frame in enumerate(frames):344 face_match = self._face_recognizer.find_target_in_frame(frame.frame_path)345 body_match = None346 if self._body_recognizer and not face_match:347 body_match = self._body_recognizer.find_target_in_frame(frame.frame_path)348 349 if face_match:350 person_scores.append(face_match.similarity)351 elif body_match:352 person_scores.append(body_match.similarity * 0.8) # Lower confidence353 else:354 person_scores.append(0.0)355 356 self._update_progress(357 PipelineStage.DETECTING_PERSON,358 (i + 1) / len(frames),359 f"Checking frame {i+1}/{len(frames)}"360 )361 except Exception as e:362 logger.warning(f"Person detection failed: {e}")363 self._update_progress(PipelineStage.DETECTING_PERSON, 1.0, "Person detection complete")364 365 # Motion analysis366 self._update_progress(PipelineStage.ANALYZING_MOTION, 0.0, "Analyzing motion...")367 motion_scores = self._compute_motion_scores(frames)368 # Fallback to visual estimation if motion detector failed or unavailable369 if not motion_scores and visual_features:370 motion_scores = self._estimate_motion_from_visual(visual_features)371 self._update_progress(PipelineStage.ANALYZING_MOTION, 1.0, "Motion analysis complete")372 373 # Scoring374 self._update_progress(PipelineStage.SCORING, 0.0, "Calculating hype scores...")375 segment_scores = self._compute_segment_scores(376 frames,377 audio_scores,378 visual_features,379 motion_scores,380 person_scores,381 clip_duration,382 )383 self._update_progress(PipelineStage.SCORING, 1.0, f"Scored {len(segment_scores)} segments")384 385 # Viral hook optimization - find best starting points386 self._update_progress(PipelineStage.OPTIMIZING_HOOKS, 0.0, "Finding viral hooks...")387 candidates = self._scores_to_candidates(segment_scores, clip_duration)388 389 # Detect hooks and optimize clip start times390 hooks = self._detect_viral_hooks(391 frames, audio_features, visual_features, motion_scores392 )393 optimized_candidates = self._optimize_clip_starts(394 candidates, hooks, num_clips395 )396 self._update_progress(PipelineStage.OPTIMIZING_HOOKS, 1.0, f"Optimized {len(optimized_candidates)} clip hooks")397 398 # Clip extraction399 self._update_progress(PipelineStage.EXTRACTING_CLIPS, 0.0, "Extracting clips...")400 clips = self._clip_extractor.extract_clips(401 video_path,402 self._temp_dir / "clips",403 optimized_candidates,404 num_clips=num_clips,405 )406 self._update_progress(PipelineStage.EXTRACTING_CLIPS, 1.0, f"Extracted {len(clips)} clips")407 408 # Handle fallback if no clips409 if not clips:410 logger.warning("No clips extracted, creating fallback clips")411 clips = self._clip_extractor.create_fallback_clips(412 video_path,413 self._temp_dir / "clips",414 metadata.duration,415 num_clips,416 )417 418 # Finalize419 self._update_progress(PipelineStage.FINALIZING, 0.0, "Finalizing...")420 processing_time = time.time() - self._start_time421 self._update_progress(PipelineStage.COMPLETE, 1.0, "Complete!")422 423 logger.info(f"Pipeline complete: {len(clips)} clips in {processing_time:.1f}s")424 425 return PipelineResult(426 success=True,427 clips=clips,428 metadata=metadata,429 scores=segment_scores,430 processing_time=processing_time,431 temp_dir=self._temp_dir,432 scenes=scenes,433 audio_features=audio_features,434 visual_features=visual_features,435 )436 437 except Exception as e:438 logger.error(f"Pipeline failed: {e}")439 logger.debug(traceback.format_exc())440 441 self._update_progress(PipelineStage.FAILED, 0.0, f"Error: {str(e)}")442 443 return PipelineResult(444 success=False,445 error_message=str(e),446 processing_time=time.time() - self._start_time,447 temp_dir=self._temp_dir,448 )449 450 def _initialize_components(451 self,452 domain: str,453 person_filter: bool,454 ) -> None:455 """Initialize pipeline components."""456 logger.info("Initializing pipeline components...")457 458 # Core components (always needed)459 self._video_processor = VideoProcessor()460 self._scene_detector = SceneDetector(461 threshold=self.config.processing.scene_threshold462 )463 self._frame_sampler = FrameSampler(464 self._video_processor,465 self.config.processing,466 )467 self._clip_extractor = ClipExtractor(468 self._video_processor,469 self.config.processing,470 )471 472 # Audio analyzer473 self._audio_analyzer = AudioAnalyzer(474 self.config.model,475 use_advanced=self.config.model.use_advanced_audio,476 )477 478 # Visual analyzer (may fail to load)479 try:480 self._visual_analyzer = VisualAnalyzer(481 self.config.model,482 load_model=True,483 )484 except Exception as e:485 logger.warning(f"Visual analyzer not available: {e}")486 self._visual_analyzer = None487 488 # Motion detector (optional, falls back to visual estimation)489 try:490 self._motion_detector = MotionDetector(491 self.config.model,492 use_raft=True, # Use high-quality RAFT if available493 )494 except Exception as e:495 logger.warning(f"Motion detector not available, using visual estimation: {e}")496 self._motion_detector = None497 498 # Person recognition (only if needed)499 if person_filter:500 try:501 self._face_recognizer = FaceRecognizer(self.config.model)502 self._body_recognizer = BodyRecognizer(self.config.model)503 except Exception as e:504 logger.warning(f"Person recognition not available: {e}")505 self._face_recognizer = None506 self._body_recognizer = None507 508 # Hype scorer509 preset = get_domain_preset(domain, person_filter_enabled=person_filter)510 self._hype_scorer = HypeScorer(preset=preset)511 512 # Viral hook detector513 self._hook_detector = ViralHookDetector(domain=domain)514 515 logger.info("Components initialized")516 517 def _compute_segment_scores(518 self,519 frames: List[SampledFrame],520 audio_scores: List,521 visual_features: List[VisualFeatures],522 motion_scores: List[float],523 person_scores: List[float],524 segment_duration: float,525 ) -> List[SegmentScore]:526 """Compute hype scores for segments."""527 if not frames:528 return []529 530 # Get timestamps from frames for visual/motion/person scores531 frame_timestamps = [f.timestamp for f in frames]532 533 # Extract scores from features534 visual_scores = [f.hype_score for f in visual_features] if visual_features else None535 536 # Audio has its own timestamps (different sampling rate)537 if audio_scores:538 audio_timestamps = [s.start_time for s in audio_scores]539 audio_vals = [s.score for s in audio_scores]540 else:541 audio_timestamps = frame_timestamps542 audio_vals = None543 544 # Use audio timestamps as the master timeline (finer granularity)545 # and interpolate other scores to match546 if audio_scores and len(audio_timestamps) > len(frame_timestamps):547 master_timestamps = audio_timestamps548 549 # Interpolate visual scores to audio timestamps550 if visual_scores:551 visual_scores = self._interpolate_scores(552 frame_timestamps, visual_scores, master_timestamps553 )554 555 # Interpolate motion scores to audio timestamps556 if motion_scores:557 motion_scores = self._interpolate_scores(558 frame_timestamps, motion_scores, master_timestamps559 )560 561 # Interpolate person scores to audio timestamps562 if person_scores:563 person_scores = self._interpolate_scores(564 frame_timestamps, person_scores, master_timestamps565 )566 else:567 master_timestamps = frame_timestamps568 # Interpolate audio to frame timestamps if needed569 if audio_vals and len(audio_vals) != len(frame_timestamps):570 audio_vals = self._interpolate_scores(571 audio_timestamps, audio_vals, frame_timestamps572 )573 574 return self._hype_scorer.score_from_timeseries(575 timestamps=master_timestamps,576 visual_series=visual_scores,577 audio_series=audio_vals,578 motion_series=motion_scores if motion_scores else None,579 person_series=person_scores if person_scores else None,580 segment_duration=segment_duration,581 hop_duration=segment_duration / 3, # Overlapping segments582 )583 584 def _interpolate_scores(585 self,586 source_timestamps: List[float],587 source_scores: List[float],588 target_timestamps: List[float],589 ) -> List[float]:590 """Interpolate scores from source timestamps to target timestamps."""591 import numpy as np592 593 if not source_timestamps or not source_scores:594 return [0.0] * len(target_timestamps)595 596 # Use numpy interpolation597 return list(np.interp(target_timestamps, source_timestamps, source_scores))598 599 def _scores_to_candidates(600 self,601 scores: List[SegmentScore],602 clip_duration: float,603 ) -> List[ClipCandidate]:604 """Convert segment scores to clip candidates."""605 return [606 ClipCandidate(607 start_time=s.start_time,608 end_time=min(s.start_time + clip_duration, s.end_time),609 hype_score=s.combined_score,610 visual_score=s.visual_score,611 audio_score=s.audio_score,612 motion_score=s.motion_score,613 person_score=s.person_score,614 )615 for s in scores616 ]617 618 def _compute_motion_scores(619 self,620 frames: List[SampledFrame],621 ) -> List[float]:622 """623 Compute motion scores using MotionDetector or fallback to visual estimation.624 625 Args:626 frames: Sampled frames with paths and timestamps627 628 Returns:629 List of motion scores (0-1) for each frame630 """631 if not frames:632 return []633 634 # Use real motion detector if available635 if self._motion_detector is not None and len(frames) >= 2:636 try:637 import cv2638 639 motion_scores = []640 641 # Load frames and compute motion between consecutive pairs642 prev_frame = None643 for i, frame in enumerate(frames):644 curr_frame = cv2.imread(str(frame.frame_path))645 646 if prev_frame is not None and curr_frame is not None:647 motion_result = self._motion_detector.analyze_motion(648 prev_frame, curr_frame, timestamp=frame.timestamp649 )650 motion_scores.append(motion_result.magnitude)651 else:652 # First frame has no motion score653 if i == 0:654 motion_scores.append(0.0)655 656 prev_frame = curr_frame657 658 logger.info(f"Computed motion scores for {len(motion_scores)} frames using RAFT/Farneback")659 return motion_scores660 661 except Exception as e:662 logger.warning(f"Motion detection failed, falling back to visual estimation: {e}")663 664 # Fallback: estimate from visual features (requires visual_features from caller)665 # Return empty list - will be filled by visual estimation in scoring666 logger.info("Using visual estimation for motion scores")667 return []668 669 def _estimate_motion_from_visual(670 self,671 visual_features: List[VisualFeatures],672 ) -> List[float]:673 """Estimate motion scores from visual analysis (fallback)."""674 if not visual_features:675 return []676 677 # Use action type as motion proxy678 motion_map = {679 "action": 0.9,680 "celebration": 0.8,681 "performance": 0.7,682 "reaction": 0.6,683 "speech": 0.3,684 "calm": 0.1,685 "transition": 0.5,686 "other": 0.4,687 }688 689 return [motion_map.get(f.action_detected, 0.4) for f in visual_features]690 691 def _apply_custom_prompt_boost(692 self,693 visual_features: List[VisualFeatures],694 custom_results: List[Dict],695 ) -> None:696 """697 Boost visual scores based on custom prompt responses.698 699 Analyzes custom prompt responses and boosts hype scores for frames700 where the response indicates a match with the user's criteria.701 702 Args:703 visual_features: Visual features to modify (in-place)704 custom_results: Results from custom prompt analysis705 """706 if not custom_results or len(custom_results) != len(visual_features):707 return708 709 # Keywords that indicate positive matches710 positive_keywords = [711 "yes", "true", "found", "detected", "present", "visible",712 "showing", "contains", "includes", "displays", "features",713 "action", "exciting", "highlight", "important", "key",714 "peak", "climax", "intense", "dramatic", "significant",715 ]716 717 for i, (features, custom) in enumerate(zip(visual_features, custom_results)):718 response = custom.get("response", "").lower()719 720 # Check for positive indicators721 match_score = 0.0722 for keyword in positive_keywords:723 if keyword in response:724 match_score += 0.1725 726 # Cap the boost at 50%727 boost = min(0.5, match_score)728 729 if boost > 0:730 # Boost the hype score731 original_score = features.hype_score732 features.hype_score = min(1.0, features.hype_score * (1 + boost))733 logger.debug(734 f"Frame {i}: custom prompt boost {boost:.2f} "735 f"({original_score:.2f} -> {features.hype_score:.2f})"736 )737 738 def _detect_viral_hooks(739 self,740 frames: List[SampledFrame],741 audio_features: List[AudioFeatures],742 visual_features: List[VisualFeatures],743 motion_scores: List[float],744 ) -> List[HookSignal]:745 """746 Detect viral hook moments from all available signals.747 748 Args:749 frames: Sampled frames with timestamps750 audio_features: Audio analysis results751 visual_features: Visual analysis results752 motion_scores: Motion intensity scores753 754 Returns:755 List of detected hook signals756 """757 if not self._hook_detector:758 return []759 760 # Prepare timestamps761 frame_timestamps = [f.timestamp for f in frames]762 763 # Prepare audio signals764 audio_timestamps = [af.timestamp for af in audio_features] if audio_features else []765 audio_energy = [af.rms_energy for af in audio_features] if audio_features else None766 audio_flux = [af.spectral_flux for af in audio_features] if audio_features else None767 audio_centroid = [af.spectral_centroid for af in audio_features] if audio_features else None768 769 # Prepare visual signals770 visual_scores = [vf.hype_score for vf in visual_features] if visual_features else None771 emotions = [vf.emotion for vf in visual_features] if visual_features else None772 actions = [vf.action_detected for vf in visual_features] if visual_features else None773 774 # Use audio timestamps if available (finer granularity), else frame timestamps775 timestamps = audio_timestamps if audio_timestamps else frame_timestamps776 777 # Interpolate visual/motion to audio timeline if needed778 if audio_timestamps and visual_scores and len(visual_scores) != len(audio_timestamps):779 visual_scores = self._interpolate_scores(frame_timestamps, visual_scores, audio_timestamps)780 motion_scores = self._interpolate_scores(frame_timestamps, motion_scores, audio_timestamps) if motion_scores else None781 # For emotions/actions, we'll use nearest neighbor (keep original)782 emotions = None # Can't interpolate strings783 actions = None784 785 # Detect hooks786 hooks = self._hook_detector.detect_hooks(787 timestamps=timestamps,788 audio_energy=audio_energy,789 audio_flux=audio_flux,790 audio_centroid=audio_centroid,791 visual_scores=visual_scores,792 motion_scores=motion_scores,793 emotions=emotions,794 actions=actions,795 )796 797 logger.info(f"Detected {len(hooks)} potential viral hook moments")798 return hooks799 800 def _optimize_clip_starts(801 self,802 candidates: List[ClipCandidate],803 hooks: List[HookSignal],804 num_clips: int,805 ) -> List[ClipCandidate]:806 """807 Optimize clip start times to align with viral hooks.808 809 Args:810 candidates: Original clip candidates811 hooks: Detected hook signals812 num_clips: Number of clips to extract813 814 Returns:815 Optimized clip candidates with adjusted start times816 """817 if not hooks or not self._hook_detector:818 logger.info("No hooks detected, using original clip timings")819 return candidates820 821 optimized = []822 823 # Process top candidates824 for candidate in candidates[:num_clips * 2]: # Consider more candidates for optimization825 # Find best hook-aligned start time826 adjusted_start, best_hook = self._hook_detector.find_best_clip_start(827 clip_start=candidate.start_time,828 clip_end=candidate.end_time,829 hooks=hooks,830 allow_adjustment=3.0, # Allow up to 3 seconds earlier831 )832 833 # Create optimized candidate834 clip_duration = candidate.end_time - candidate.start_time835 836 # Boost score if we found a good hook837 hook_boost = 1.0838 if best_hook:839 hook_score = self._hook_detector.score_clip_hook_potential(840 adjusted_start, clip_duration, hooks841 )842 hook_boost = 1.0 + (hook_score * 0.3) # Up to 30% boost843 844 optimized.append(ClipCandidate(845 start_time=adjusted_start,846 end_time=adjusted_start + clip_duration,847 hype_score=candidate.hype_score * hook_boost,848 visual_score=candidate.visual_score,849 audio_score=candidate.audio_score,850 motion_score=candidate.motion_score,851 person_score=candidate.person_score,852 ))853 854 if best_hook:855 logger.debug(856 f"Clip {candidate.start_time:.1f}s -> {adjusted_start:.1f}s "857 f"(hook: {best_hook.hook_type.value}, boost: {hook_boost:.2f}x)"858 )859 860 # Re-sort by boosted score861 optimized.sort(key=lambda c: c.hype_score, reverse=True)862 863 logger.info(f"Optimized {len(optimized)} candidates with viral hooks")864 return optimized865 866 def cleanup(self) -> None:867 """Clean up temporary files and unload models."""868 if self._temp_dir:869 cleanup_temp_files(self._temp_dir)870 self._temp_dir = None871 872 if self._visual_analyzer:873 self._visual_analyzer.unload_model()874 875 logger.info("Pipeline cleanup complete")876 877 878# Export public interface879__all__ = ["PipelineOrchestrator", "PipelineResult", "PipelineProgress", "PipelineStage"]880