Aniket2012/video_analysis
0
1# ==============================================================================2# PitchPerfect AI: Enterprise-Grade Sales Coach (Single File Application)3#4# This single file contains the complete application code, including a5# self-contained dependency installer, YouTube support, JAX-based quantitative6# analysis, and a robust agentic architecture.7# ==============================================================================8 9# ==============================================================================10# DYNAMIC DEPENDENCY INSTALLATION11# This block checks for and installs missing packages.12# ==============================================================================13import sys14import subprocess15import importlib.util16import logging17 18# Configure basic logging for the installation process19logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')20 21def install_dependencies():22 """23 Checks for required packages and installs them if they are not found.24 This makes the script self-contained and removes the need for a25 separate requirements.txt and `pip install` step.26 """27 required_packages = [28 # Core application29 ("gradio", "gradio"),30 ("google.cloud.aiplatform", "google-cloud-aiplatform"),31 ("google.cloud.storage", "google-cloud-storage"),32 ("moviepy", "moviepy"),33 # For JAX and Quantitative Analysis34 ("jax", "jax"),35 ("jaxlib", "jaxlib"),36 ("librosa", "librosa"),37 ("speechrecognition", "SpeechRecognition"),38 ("whisper", "openai-whisper"),39 # For YouTube support40 ("yt_dlp", "yt-dlp"),41 ]42 43 print("="*80)44 print("PitchPerfect AI: Checking for required dependencies...")45 print("="*80)46 47 for import_name, install_name in required_packages:48 spec = importlib.util.find_spec(import_name)49 if spec is None:50 print(f"INFO: Package '{install_name}' not found. Attempting to install...")51 try:52 subprocess.check_call([sys.executable, "-m", "pip", "install", "--quiet", install_name])53 print(f"INFO: Successfully installed '{install_name}'.")54 except subprocess.CalledProcessError:55 print(f"ERROR: Failed to install '{install_name}'. Please install it manually using:\n"56 f"pip install {install_name}")57 sys.exit(1)58 else:59 print(f"INFO: Dependency '{install_name}' is already installed.")60 61 print("="*80)62 print("All dependencies are satisfied. Starting the application.")63 print("="*80)64 65# Run the dependency check and installation before anything else.66install_dependencies()67 68 69# ==============================================================================70# File: README.md (Instructions are now part of the script's execution)71# ==============================================================================72"""73# PitchPerfect AI: Enterprise-Grade Sales Coach74 75This application provides AI-powered feedback on sales pitches using Google's most advanced multimodal AI, all managed through the Vertex AI platform. It analyzes your content, vocal delivery, and visual presence to give you actionable insights for improvement.76 77This advanced version includes:78- Support for local video uploads and YouTube URLs.79- Quantitative vocal analysis powered by JAX for high performance.80- An agentic architecture where specialized tools (YouTube Downloader, JAX Analyzer) work in concert with the Gemini 1.5 Pro model.81- A self-contained dependency installer for simplified setup.82 83## ๐ Prerequisites84 851. A Google Cloud Platform (GCP) project with billing enabled.862. The Vertex AI API and Cloud Storage API enabled in your GCP project.873. The `gcloud` CLI installed and authenticated on your local machine.88 89## ์
์
90 911. **Create a Google Cloud Storage (GCS) Bucket:**92 * In your GCP project, create a new GCS bucket. It must have a globally unique name.93 * **Example name:** `your-project-id-pitch-videos`94 952. **Authenticate with Google Cloud:**96 Run the following command in your terminal and follow the prompts. This sets up Application Default Credentials (ADC).97 ```bash98 gcloud auth application-default login99 ```100 *Note: The user/principal needs `Storage Object Admin` and `Vertex AI User` roles.*101 1023. **Configure Project Details:**103 * In this file, scroll down to the "CONFIGURATION" section.104 * Set your `GCP_PROJECT_ID`, `GCP_LOCATION`, and `GCS_BUCKET_NAME`.105 1064. **Run the Application:**107 Simply run the script. It will automatically install any missing dependencies.108 ```bash109 python app.py110 ```111 This will launch a Gradio web server. **Look for a public URL ending in `.gradio.live` in the output and open it in your browser.**112"""113 114# ==============================================================================115# IMPORTS116# ==============================================================================117import json118import uuid119import os120import re121from typing import Dict, Any122import gradio as gr123import vertexai124from google.cloud import storage125from vertexai.generative_models import (126 GenerativeModel, Part, GenerationConfig,127 HarmCategory, HarmBlockThreshold128)129 130# Third-party imports for advanced features131import yt_dlp132import librosa133import numpy as np134import whisper135import jax136import jax.numpy as jnp137from moviepy.editor import VideoFileClip138 139 140# ==============================================================================141# CONFIGURATION142# ==============================================================================143# --- GCP and Vertex AI Configuration ---144GCP_PROJECT_ID = "aniket-personal"145GCP_LOCATION = "us-central1"146 147# --- GCS Configuration ---148GCS_BUCKET_NAME = "ghiblify"149 150# --- Model Configuration ---151MODEL_GEMINI_PRO = "gemini-1.5-pro-preview-0514"152 153# --- Example Videos ---154# These are publicly accessible videos for demonstration purposes.155EXAMPLE_VIDEOS = [156 ["Confident Business Presentation", "https://storage.googleapis.com/pitchperfect-ai-examples/business_pitch_example.mp4"],157 ["Casual Tech Talk", "https://storage.googleapis.com/pitchperfect-ai-examples/tech_talk_example.mp4"],158]159 160# --- Schemas for Controlled Generation (as Dictionaries) ---161FEEDBACK_ITEM_SCHEMA = {162 "type": "object",163 "properties": {164 "score": {"type": "integer", "minimum": 1, "maximum": 10},165 "feedback": {"type": "string"}166 },167 "required": ["score", "feedback"]168}169HOLISTIC_ANALYSIS_SCHEMA = {170 "type": "object",171 "properties": {172 "content_analysis": {"type": "object", "properties": {"clarity": FEEDBACK_ITEM_SCHEMA, "structure": FEEDBACK_ITEM_SCHEMA, "value_proposition": FEEDBACK_ITEM_SCHEMA, "cta": FEEDBACK_ITEM_SCHEMA}},173 "vocal_analysis": {"type": "object", "properties": {"pacing": FEEDBACK_ITEM_SCHEMA, "vocal_variety": FEEDBACK_ITEM_SCHEMA, "confidence_energy": FEEDBACK_ITEM_SCHEMA, "clarity_enunciation": FEEDBACK_ITEM_SCHEMA}},174 "visual_analysis": {"type": "object", "properties": {"eye_contact": FEEDBACK_ITEM_SCHEMA, "body_language": FEEDBACK_ITEM_SCHEMA, "facial_expressions": FEEDBACK_ITEM_SCHEMA}}175 },176 "required": ["content_analysis", "vocal_analysis", "visual_analysis"]177}178FINAL_SYNTHESIS_SCHEMA = {179 "type": "object",180 "properties": {181 "key_strengths": {"type": "string"},182 "growth_opportunities": {"type": "string"},183 "executive_summary": {"type": "string"}184 },185 "required": ["key_strengths", "growth_opportunities", "executive_summary"]186}187 188# --- Enhanced Prompts ---189PROMPT_HOLISTIC_VIDEO_ANALYSIS = """190You are an expert sales coach. Analyze the provided video and the supplementary quantitative metrics to generate a structured, holistic feedback report. Your output MUST strictly conform to the provided JSON schema, including the 1-10 score range.191 192**Quantitative Metrics (for additional context):**193{quantitative_metrics_json}194 195**Evaluation Framework (Analyze the video directly):**1961. **Content & Structure:** Analyze clarity, flow, value proposition, and the call to action.1972. **Vocal Delivery:** Analyze pacing, vocal variety, confidence, energy, and enunciation. Use the quantitative metrics to inform your qualitative assessment.1983. **Visual Delivery:** Analyze eye contact, body language, and facial expressions.199 200Provide specific examples from the video to support your points.201"""202 203PROMPT_FINAL_SYNTHESIS = """204You are a senior executive coach. Synthesize the provided detailed analysis data into a high-level summary. Your output MUST strictly conform to the provided JSON schema.205 206- "key_strengths" should be a single string with bullet points (e.g., "- Point one\\n- Point two").207- "growth_opportunities" should be a single string, formatted similarly.208- "executive_summary" should be a single string paragraph.209 210**Detailed Analysis Data:**211---212{full_analysis_json}213---214"""215 216# ==============================================================================217# AGENT TOOLKIT218# ==============================================================================219class YouTubeDownloaderTool:220 """A tool to download a YouTube video to a local path."""221 def run(self, url: str, output_dir: str = "temp_downloads") -> str:222 if not os.path.exists(output_dir):223 os.makedirs(output_dir)224 225 filepath = os.path.join(output_dir, f"{uuid.uuid4()}.mp4")226 ydl_opts = {227 'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]/best[ext=mp4]/best',228 'outtmpl': filepath,229 'quiet': True,230 }231 with yt_dlp.YoutubeDL(ydl_opts) as ydl:232 ydl.download([url])233 return filepath234 235class QuantitativeAudioTool:236 """A tool for performing objective, numerical analysis on an audio track."""237 class JAXAudioProcessor:238 """A nested class demonstrating JAX for high-performance audio processing."""239 def __init__(self):240 self.jit_rms_energy = jax.jit(self._calculate_rms_energy)241 @staticmethod242 @jax.jit243 def _calculate_rms_energy(waveform: jnp.ndarray) -> jnp.ndarray:244 return jnp.sqrt(jnp.mean(jnp.square(waveform)))245 def analyze_energy_variation(self, waveform_np):246 if waveform_np is None or waveform_np.size == 0: return 0.0247 waveform_jnp = jnp.asarray(waveform_np)248 frame_length, hop_length = 2048, 512249 num_frames = (waveform_jnp.shape[0] - frame_length) // hop_length250 start_positions = jnp.arange(num_frames) * hop_length251 offsets = jnp.arange(frame_length)252 frame_indices = start_positions[:, None] + offsets[None, :]253 frames = waveform_jnp[frame_indices]254 frame_energies = jax.vmap(self.jit_rms_energy)(frames)255 return float(jnp.std(frame_energies))256 257 def __init__(self):258 self.jax_processor = self.JAXAudioProcessor()259 # Lazily load the whisper model to avoid loading it if not needed260 self._whisper_model = None261 262 @property263 def whisper_model(self):264 if self._whisper_model is None:265 logging.info("Loading whisper model 'base.en' for the first time...")266 self._whisper_model = whisper.load_model("base.en")267 logging.info("Whisper model loaded.")268 return self._whisper_model269 270 def run(self, video_path: str, output_dir: str = "temp_output"):271 if not os.path.exists(output_dir): os.makedirs(output_dir)272 video = None273 try:274 video = VideoFileClip(video_path)275 276 if video.audio is None:277 raise ValueError("The provided video file does not contain an audio track, or it could not be decoded. Analysis cannot proceed.")278 279 audio_path = os.path.join(output_dir, f"audio_{uuid.uuid4()}.wav")280 video.audio.write_audiofile(audio_path, codec='pcm_s16le', fps=16000, logger=None)281 282 transcript_result = self.whisper_model.transcribe(audio_path, fp16=False)283 word_count = len(transcript_result['text'].split())284 duration = video.duration285 pace = (word_count / duration) * 60 if duration > 0 else 0286 287 y, sr = librosa.load(audio_path, sr=16000)288 energy_variation = self.jax_processor.analyze_energy_variation(y)289 290 os.remove(audio_path)291 292 return {293 "speaking_pace_wpm": round(pace, 2),294 "vocal_energy_variation": round(energy_variation, 4),295 }296 finally:297 if video:298 video.close()299 300# ==============================================================================301# VERTEX AI MANAGER CLASS302# ==============================================================================303class VertexAIManager:304 def __init__(self):305 vertexai.init(project=GCP_PROJECT_ID, location=GCP_LOCATION)306 self.model = GenerativeModel(MODEL_GEMINI_PRO)307 308 def run_multimodal_analysis(self, video_gcs_uri: str, prompt: str) -> dict:309 video_part = Part.from_uri(uri=video_gcs_uri, mime_type="video/mp4")310 contents = [video_part, prompt]311 config = GenerationConfig(response_schema=HOLISTIC_ANALYSIS_SCHEMA, temperature=0.2, response_mime_type="application/json")312 response = self.model.generate_content(contents, generation_config=config)313 return json.loads(response.text)314 315 def run_synthesis(self, prompt: str) -> dict:316 config = GenerationConfig(response_schema=FINAL_SYNTHESIS_SCHEMA, temperature=0.3, response_mime_type="application/json")317 response = self.model.generate_content(prompt, generation_config=config)318 return json.loads(response.text)319 320# ==============================================================================321# AGENT CLASS322# ==============================================================================323class PitchAnalyzerAgent:324 def __init__(self):325 self.vertex_manager = VertexAIManager()326 self.storage_client = storage.Client(project=GCP_PROJECT_ID)327 self.youtube_tool = YouTubeDownloaderTool()328 self.quant_tool = QuantitativeAudioTool()329 self._check_bucket()330 331 def _check_bucket(self):332 logging.info(f"Checking access to GCS Bucket: {GCS_BUCKET_NAME}")333 self.storage_client.get_bucket(GCS_BUCKET_NAME)334 logging.info("GCS Bucket access confirmed.")335 336 337 def _upload_to_gcs(self, path: str) -> str:338 bucket = self.storage_client.bucket(GCS_BUCKET_NAME)339 blob_name = f"pitch-videos/{uuid.uuid4()}.mp4"340 blob = bucket.blob(blob_name)341 blob.upload_from_filename(path)342 logging.info(f"Successfully uploaded video to gs://{GCS_BUCKET_NAME}/{blob_name}")343 return f"gs://{GCS_BUCKET_NAME}/{blob_name}"344 345 def _delete_from_gcs(self, gcs_uri: str):346 try:347 bucket_name, blob_name = gcs_uri.replace("gs://", "").split("/", 1)348 self.storage_client.bucket(bucket_name).blob(blob_name).delete()349 logging.info(f"Successfully deleted GCS object: {gcs_uri}")350 except Exception as e:351 logging.warning(f"Failed to delete GCS object {gcs_uri}: {e}")352 353 def run_analysis_pipeline(self, video_path_or_url: str, progress_callback):354 local_video_path = None355 video_gcs_uri = None356 is_youtube_download = False357 try:358 if re.match(r"^(https?://)?(www\.)?(youtube\.com|youtu\.?be)/.+$", video_path_or_url):359 progress_callback(0.1, "Downloading video from YouTube...")360 local_video_path = self.youtube_tool.run(video_path_or_url)361 is_youtube_download = True362 else:363 local_video_path = video_path_or_url364 365 progress_callback(0.3, "Performing JAX-based quantitative analysis...")366 quant_metrics = self.quant_tool.run(local_video_path)367 368 progress_callback(0.5, "Uploading video to secure Cloud Storage...")369 video_gcs_uri = self._upload_to_gcs(local_video_path)370 371 progress_callback(0.7, "Gemini 1.5 Pro is analyzing the video...")372 analysis_prompt = PROMPT_HOLISTIC_VIDEO_ANALYSIS.format(quantitative_metrics_json=json.dumps(quant_metrics, indent=2))373 multimodal_analysis = self.vertex_manager.run_multimodal_analysis(video_gcs_uri, analysis_prompt)374 375 progress_callback(0.9, "Synthesizing final report...")376 synthesis_prompt = PROMPT_FINAL_SYNTHESIS.format(full_analysis_json=json.dumps(multimodal_analysis, indent=2))377 final_summary = self.vertex_manager.run_synthesis(synthesis_prompt)378 379 return {"quantitative_metrics": quant_metrics, "multimodal_analysis": multimodal_analysis, "executive_summary": final_summary}380 except Exception as e:381 logging.error(f"Analysis pipeline failed: {e}", exc_info=True)382 return {"error": str(e)}383 finally:384 if video_gcs_uri:385 self._delete_from_gcs(video_gcs_uri)386 # Only delete the local file if it was downloaded from YouTube387 if is_youtube_download and local_video_path and os.path.exists(local_video_path):388 os.remove(local_video_path)389 logging.info(f"Deleted temporary YouTube download: {local_video_path}")390 391# ==============================================================================392# UI FORMATTING HELPER393# ==============================================================================394def format_feedback_markdown(analysis: dict) -> str:395 if not analysis or "error" in analysis:396 error_message = analysis.get('error', 'Unknown error.')397 logging.error(f"Displaying analysis failure to user: {error_message}")398 return f"## Analysis Failed ๐\n\n**Reason:** {error_message}"399 400 summary = analysis.get('executive_summary', {})401 metrics = analysis.get('quantitative_metrics', {})402 ai_analysis = analysis.get('multimodal_analysis', {})403 404 def get_pace_rating(wpm):405 if wpm == 0: return "N/A (No speech detected)"406 if wpm < 120: return "Slow / Deliberate"407 if wpm <= 160: return "Conversational"408 return "Fast-Paced"409 410 def get_energy_rating(variation):411 if variation == 0: return "N/A"412 if variation < 0.02: return "Consistent / Monotonous"413 if variation <= 0.05: return "Moderately Dynamic"414 return "Highly Dynamic & Engaging"415 416 wpm = metrics.get('speaking_pace_wpm', 0)417 energy_var = metrics.get('vocal_energy_variation', 0)418 pace_rating = get_pace_rating(wpm)419 energy_rating = get_energy_rating(energy_var)420 421 metrics_md = f"""422- **Speaking Pace:** **{wpm} WPM** *(Rating: {pace_rating})*423 - *This measures the number of words spoken per minute. A typical conversational pace is between 120-160 WPM.*424- **Vocal Energy Variation:** **{energy_var:.4f}** *(Rating: {energy_rating})*425 - *This measures the standard deviation of your vocal loudness. A higher value indicates a more dynamic and engaging vocal range, while a very low value suggests a monotonous delivery.*426 """427 428 def format_ai_item(title, data):429 if not data or "score" not in data: return f"**{title}:**\n> Analysis not available.\n\n"430 raw_score = data.get('score', 0); score = max(1, min(10, raw_score))431 stars = "๐ข" * score + "โช๏ธ" * (10 - score)432 feedback = data.get('feedback', 'No feedback.').replace('\n', '\n> ')433 return f"**{title}:** `{stars} [{score}/10]`\n\n> {feedback}\n\n"434 435 content = ai_analysis.get('content_analysis', {}); vocal = ai_analysis.get('vocal_analysis', {}); visual = ai_analysis.get('visual_analysis', {})436 437 return f"""438# PitchPerfect AI Analysis Report ๐439## ๐ Executive Summary440### Key Strengths441{summary.get('key_strengths', '- N/A')}442### High-Leverage Growth Opportunities443{summary.get('growth_opportunities', '- N/A')}444### Final Verdict445> {summary.get('executive_summary', 'N/A')}446---447## ๐ Quantitative Metrics Explained (via JAX & Whisper)448{metrics_md}449---450## ๐ง AI Multimodal Analysis (via Gemini 1.5 Pro)451### I. Content & Structure452{format_ai_item("Clarity", content.get('clarity'))}453{format_ai_item("Structure & Flow", content.get('structure'))}454{format_ai_item("Value Proposition", content.get('value_proposition'))}455{format_ai_item("Call to Action (CTA)", content.get('cta'))}456<hr style="border:1px solid #ddd">457 458### II. Vocal Delivery459{format_ai_item("Pacing", vocal.get('pacing'))}460{format_ai_item("Vocal Variety", vocal.get('vocal_variety'))}461{format_ai_item("Confidence & Energy", vocal.get('confidence_energy'))}462{format_ai_item("Clarity & Enunciation", vocal.get('clarity_enunciation'))}463<hr style="border:1px solid #ddd">464 465### III. Visual Delivery466{format_ai_item("Eye Contact", visual.get('eye_contact'))}467{format_ai_item("Body Language", visual.get('body_language'))}468{format_ai_item("Facial Expressions", visual.get('facial_expressions'))}469"""470 471# ==============================================================================472# GRADIO APPLICATION473# ==============================================================================474if __name__ == "__main__":475 pitch_agent = None476 try:477 # Initialize the agent only if the script is run directly.478 pitch_agent = PitchAnalyzerAgent()479 except Exception as e:480 logging.fatal(f"Failed to initialize agent during startup: {e}", exc_info=True)481 # Display the error in a simplified Gradio interface if initialization fails482 with gr.Blocks(theme=gr.themes.Soft()) as demo:483 gr.Markdown(f"""484 # ## ๐ด FATAL ERROR485 Could not initialize the PitchPerfect AI Agent. This is likely due to a configuration issue.486 Please check the following:487 1. You have authenticated with `gcloud auth application-default login`.488 2. The GCP Project ID (`{GCP_PROJECT_ID}`) and GCS Bucket (`{GCS_BUCKET_NAME}`) are correct and accessible.489 3. The necessary APIs (Vertex AI, Cloud Storage) are enabled in your project.490 491 **Error Details:**492 ```493 {e}494 ```495 """)496 demo.launch()497 sys.exit(1)498 499 500 def run_analysis_pipeline_interface(video_path, url_path, progress=gr.Progress(track_tqdm=True)):501 """Interface function for Gradio to call the agent's pipeline."""502 if not pitch_agent:503 return "## FATAL ERROR: Application not initialized. Check logs and configuration."504 505 input_path = url_path if url_path and url_path.strip() else video_path506 if not input_path:507 return "## No Video Provided\nPlease upload a video file or enter a valid YouTube URL to begin."508 509 # Clear the other input field to avoid confusion510 if url_path:511 video_path = None512 else:513 url_path = None514 515 analysis_result = pitch_agent.run_analysis_pipeline(input_path, progress.update)516 return format_feedback_markdown(analysis_result)517 518 # Define the Gradio UI519 with gr.Blocks(theme=gr.themes.Soft(primary_hue="teal", secondary_hue="orange")) as demo:520 gr.Markdown("# **PitchPerfect AI**: Your Enterprise-Grade Sales Coach ๐")521 with gr.Row():522 with gr.Column(scale=1):523 video_uploader = gr.Video(label="Upload Your Pitch", sources=["upload"])524 gr.Markdown("<center>--- **OR** ---</center>")525 youtube_url = gr.Textbox(label="Enter YouTube URL", placeholder="e.g., https://www.youtube.com/watch?v=...")526 analyze_button = gr.Button("Analyze My Pitch ๐ง ", variant="primary")527 gr.Examples(examples=EXAMPLE_VIDEOS, inputs=youtube_url, label="Example Pitches (Click to Use)")528 with gr.Column(scale=2):529 analysis_output = gr.Markdown(label="Your Feedback Report", value="### Your detailed report will appear here...")530 531 # Connect the button click to the analysis function532 analyze_button.click(533 fn=run_analysis_pipeline_interface,534 inputs=[video_uploader, youtube_url],535 outputs=analysis_output536 )537 538 # Launch the Gradio application539 demo.launch(debug=True, share=True)