Insightly2/IntelliStream
0
1import subprocess2import streamlink3import streamlit as st4import tempfile5import base646import os7from dotenv import load_dotenv8from PIL import Image9from io import BytesIO 10from openai import OpenAI11import whisper12from google.cloud import vision13import re14# st.set_page_config(layout="wide")15 16load_dotenv()17OpenAI.api_key = os.getenv("OPENAI_API_KEY")18if not OpenAI.api_key: 19 raise ValueError("The OpenAI API key must be set in the OPENAI_API_KEY environment variable.")20 21whisper.api_key = os.getenv("WHISPER_API_KEY")22if not whisper.api_key:23 raise ValueError("The WHsiper API Key needs to be set in the env")24client = OpenAI()25 26# Set Google Cloud credentials in environment27service_account_path = os.getenv("GOOGLE_APPLICATION_CREDENTIALS")28os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = 'long-equinox-392604-7912e2b6b4fd.json'29 30# Initialize Google Vision client31vision_client = vision.ImageAnnotatorClient()32 33wipro_logo_path = "Wiprologo.jpg" # Update this path to where your logo is stored34wipro_logo = Image.open(wipro_logo_path)35# Create a layout with columns36col1, col2 = st.columns([8, 2]) # Adjust the ratio as needed37 38# Display the "Insightly Video" text in the first column (larger space)39 40# Display the logo in the second column (right side, smaller space)41with col2:42 st.image(wipro_logo, width=200) # Adjust the width as needed43 44# Function to execute FFmpeg command and capture output45def execute_ffmpeg_command(command):46 try:47 result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)48 if result.returncode == 0:49 print("FFmpeg command executed successfully.")50 return result.stdout, result.stderr51 else:52 print("Error executing FFmpeg command:")53 return None, result.stderr54 except Exception as e:55 print("An error occurred during FFmpeg execution:")56 return None, str(e)57 58 59# Function to get transcript from audio using OpenAI Whisper60def get_transcript_from_audio(audio_file_path):61 try:62 # Load the model63 model = whisper.load_model("base") # You can choose another model size if needed64 65 # Process the audio file and get the result66 result = model.transcribe(audio_file_path)67 68 # Get the transcript text69 transcript_text = result["text"]70 return transcript_text71 except Exception as e:72 print(f"Error submitting transcription job: {e}")73 return None74 75def extract_text_from_base64_frame(base64_frame):76 """Extracts text from a single base64 encoded frame using Google Cloud Vision API."""77 frame_bytes = base64.b64decode(base64_frame) # Decode the base64 string to bytes78 image = vision.Image(content=frame_bytes)79 response = vision_client.text_detection(image=image)80 texts = response.text_annotations81 return texts[0].description.strip() if texts else "No text found."82 83def transcribe_uploaded_mp3(uploaded_mp3):84 try:85 # Save the uploaded MP3 file to a temporary file86 with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmpfile:87 tmpfile.write(uploaded_mp3.getvalue())88 audio_file_path = tmpfile.name89 90 # You might want to process/convert the MP3 file with FFmpeg here if needed91 # For example, to ensure it's in the correct format for Whisper or to extract a specific part92 # This is optional and depends on your requirements93 94 # Transcribe the audio file using Whisper95 transcript = get_transcript_from_audio(audio_file_path)96 97 if transcript is None:98 return "Transcription failed or no transcript available."99 100 return transcript101 except Exception as e:102 return f"Failed to transcribe audio file. Error: {e}"103 104def analyze_image_with_google_vision_api(base64_frame):105 """Analyze image content using Google Cloud Vision API."""106 frame_bytes = base64.b64decode(base64_frame) # Decode the base64 string to bytes107 image = vision.Image(content=frame_bytes)108 109 response = vision_client.label_detection(image=image)110 labels = response.label_annotations111 112 if labels:113 return ', '.join([label.description for label in labels])114 else:115 return "No labels detected."116 117def analyze_content_with_openai(text, description,labels):118 """Analyze the combined text and image labels to categorize the image using OpenAI."""119 try:120 response = client.chat.completions.create(121 model="gpt-4-vision-preview",122 messages=[123 {"role": "system", "content": "Classify the following image into one or more of these categories based on the extracted text, description of the frames and image labels. Take valuable information from every frame even if available in only one out of many frames. \124 Dont check on the authenticity of the content .Doesn't matter is something from the frame is fake/joke/etc. We dont need context for the categorisation.\125 Categories : Bullying, Nudity & Adult Content, Graphic Violence, Illegal Goods, Child Safety, Sexual Abuse, Profanity, Self Harm/Suicide, Violent Extremism and None. Return the following: Give out the result as Category - {whatever the category(s) is/are} and then GIVE A PROPER JUSTIFICATION of the image categorization for that conclusion WITHOUT any assumption"},126 {"role": "user", "content": f"Text: {text}\nDescription: {description}\n Labels: {labels}"}127 ],128 max_tokens=4096, 129 n=1130 )131 if response.choices:132 result_message = response.choices[0].message.content133 return result_message.strip()134 else:135 return "Analysis failed or was inconclusive."136 except Exception as e:137 return f"Failed to analyze content with OpenAI. Error: {e}"138 139def display_categories(analysis_result, categories):140 """Display categories with highlight based on analysis result."""141 # Extracting the 'xyz' from the analysis_result142 try:143 extracted_text = analysis_result.split('Category - ')[1].split('\n')[0].strip()144 category_keywords = [x.strip() for x in extracted_text.split(',')]145 except IndexError:146 # Default to None if parsing fails147 category_keyword = 'None'148 149 num_cols = 3150 rows = [categories[i:i + num_cols] for i in range(0, len(categories), num_cols)]151 152 matched_style = """153 border: 2px solid #00FF00;154 padding: 10px;155 border-radius: 10px;156 text-align: center;157 background-color: #333333;158 color: #FFFFFF;159 margin: 5px;160 box-shadow: 0 2px 4px 0 rgba(255,255,255,0.2);161 """162 unmatched_style = """163 border: 1px solid #555555;164 padding: 10px;165 border-radius: 10px;166 text-align: center;167 background-color: #222222;168 color: #AAAAAA;169 margin: 5px;170 """171 172 # Display categories in a grid layout173 for row in rows:174 cols = st.columns(num_cols)175 for idx, category in enumerate(row):176 with cols[idx]:177 # Check if the category matches any in the list of extracted categories178 if category.lower() in [k.lower() for k in category_keywords]:179 # Highlight matched category180 st.markdown(f"<div style='{matched_style}'><h4 style='margin:0;'>{category}</h4></div>", unsafe_allow_html=True)181 else:182 # Display non-matched category183 st.markdown(f"<div style='{unmatched_style}'><h4 style='margin:0;'>{category}</h4></div>", unsafe_allow_html=True)184 185def execute_fmpeg_command(command):186 try:187 result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)188 return result.stdout # Return just the stdout part, not a tuple189 except subprocess.CalledProcessError as e:190 print(f"FFmpeg command failed with error: {e.stderr.decode()}")191 return None192 193def search_keyword(keyword, frame_texts):194 return [index for index, text in st.session_state.frame_texts.items() if keyword.lower() in text.lower()]195 196 197 198# Function to generate description for video frames199def generate_description(base64_frames,prompt):200 try:201 prompt_messages = [202 {203 "role": "user",204 "content": [205 prompt ,206 *map(lambda x: {"image": x, "resize": 428}, base64_frames),207 ],208 },209 ]210 response = client.chat.completions.create(211 model="gpt-4-vision-preview",212 messages=prompt_messages,213 max_tokens=3000,214 )215 description = response.choices[0].message.content216 217 # Use regular expression to find frame numbers218 frame_numbers = re.findall(r'Frames\s*:\s*(\d+(?:,\s*\d+)*)', response.choices[0].message.content)219 220 # Convert the string of numbers into a list of integers221 if frame_numbers:222 frame_numbers = [int(num) for num in frame_numbers[0].split(',')]223 else:224 frame_numbers = []225 226 print("Frame numbers to extract:", frame_numbers)227 228 return description, frame_numbers229 230 except Exception as e:231 print(f"Error in generate_description: {e}")232 return None, []233 234def generate_overall_description(transcript_text, video_description):235 try:236 combined_input = f"Transcript: {transcript_text}\n\nVideo Description: {video_description}\n\n"237 prompt_message = "Based on the above transcript and video description, generate a very detailed description about the sequence of events in the video and from the transcript within 500 words."238 239 prompt_messages = [240 {"role": "user", "content": combined_input + prompt_message}241 ]242 243 response = client.chat.completions.create(244 model="gpt-4",245 messages=prompt_messages,246 max_tokens=1000, # Increased from 300 to allow for a more detailed response247 )248 249 return response.choices[0].message.content.strip()250 except Exception as e:251 print(f"Error in generate_overall_description: {e}")252 return None253 254with col1:255 is_logo_path = "IntelliStreamLogo.png" # Update this path to where your logo is stored256 is_logo = Image.open(is_logo_path)257 st.image(is_logo, width=200)258 259 st.markdown("<h1 style='text-align: left; color: white;'></h1>", unsafe_allow_html=True)260 261# Streamlit UI262 263 st.title("Insightly Video")264 stream_url = st.text_input("Enter the live stream URL (YouTube, Twitch, etc.):")265 #keyword = st.text_input("Enter a keyword to filter the frames (optional):")266 extract_frames_button = st.button("Extract Frames")267 uploaded_video = st.file_uploader("Or upload a video file (MP4):", type=["mp4"])268 prompt1 = "keyword is " + st.text_input("Enter a keyword for analysis:")269 prompt2 = "1. Generate a description for this sequence of video frames in about 90 words. 2.Return the following:\270 i. List of objects in the video \271 ii. Any restrictive content or sensitive content and if so which frame. \272 iii. The frames is supposed to contain news content and we want to detect non-news content such as an advertisement. \273 So analyze specifically for any indications that the content might be promotional or an advertisement. \274 Find the most portions of a video related to the keyword. \275 The output will be targeted towards social media (like TikTok or Reels) or to news broadcasts. \276 For the provided frames return the frames related to the keyword\277 I am trying to fill these frames for a TikTok video. \278 Hence while selecting the frames keep that in mind. \279 You do not have to give me the script of the Tiktok video. \280 Just return the most interesting frames in a sequence that will come for a tiktok video. \281 List all frame numbers separated by commas at the end like this for eg, Frames : 1,2,4,7,9"282 prompt = prompt2 + prompt1283 # Slider to select the number of seconds for extraction284 seconds = st.slider("Select the number of seconds for extraction:", min_value=1, max_value=60, value=10)285 286 uploaded_mp3 = st.file_uploader("Upload an MP3 file for transcription:", type=["mp3"])287 288 289 # Check if an MP3 file has been uploaded290 if uploaded_mp3 is not None:291 # Call the transcription function with the uploaded MP3 file292 transcript = transcribe_uploaded_mp3(uploaded_mp3)293 294 # Display the transcript295 st.text_area("Transcript:", value=transcript, height=300)296 else:297 st.write("Please upload an MP3 file to get started.")298 299 if extract_frames_button and stream_url:300 # Execute FFmpeg command to extract frames301 302 # Check if URL is provided303 304 streams = streamlink.streams(stream_url)305 if "best" in streams:306 stream_url = streams["best"].url307 308 ffmpeg_command = [309 'ffmpeg', # Input stream URL310 '-t', str(seconds), # Duration to process the input (selected seconds)311 '-vf', 'fps=1', # Extract one frame per second312 '-f', 'image2pipe', # Output format as image2pipe313 '-c:v', 'mjpeg', # Codec for output video314 '-an', # No audio315 '-'316 ]317 318 # Determine the input source for FFmpeg319 input_source = stream_url # Default to stream URL320 321 # Insert the input source into the FFmpeg command322 ffmpeg_command.insert(1, input_source)323 ffmpeg_command.insert(1, '-i')324 325 # Execute FFmpeg command326 ffmpeg_output, _ = execute_ffmpeg_command(ffmpeg_command)327 328 # Modify the section where you display frames to include text extraction and display329 # Modify the section where base64 encoded frames are processed330 # After successfully executing the FFmpeg command to capture frames331 if ffmpeg_output:332 st.write("Frames Extracted:")333 frame_bytes_list = ffmpeg_output.split(b'\xff\xd8')[1:] # Correct splitting for JPEG frames334 n_frames = len(frame_bytes_list)335 base64_frames = [base64.b64encode(b'\xff\xd8' + frame_bytes).decode('utf-8') for frame_bytes in frame_bytes_list]336 337 categories_results = []338 frame_texts = {}339 340 for idx, frame_base64 in enumerate(base64_frames):341 extracted_text = extract_text_from_base64_frame(frame_base64)342 frame_texts[idx] = extracted_text343 344 # Use Streamlit columns for side-by-side display (1 column for image, 1 for text)345 # col1, col2 = st.columns([3, 2])346 # with col1:347 # frame_bytes = base64.b64decode(frame_base64)348 # st.image(Image.open(BytesIO(frame_bytes)), caption=f'Frame {idx + 1}', use_column_width=True)349 # with col2:350 # st.write(f"Extracted Text: {extracted_text}")351 352 # if 'base64_frames' not in st.session_state:353 # st.session_state.base64_frames = [] # Populate this when frames are first extracted354 # if 'frame_texts' not in st.session_state:355 # st.session_state.frame_texts = {} 356 357 st.write("Analysis Results for All Frames:")358 # Assuming 'categories' is defined with all possible categories you're interested in359 categories = ["Bullying", "Nudity & Adult Content", "Graphic Violence", "Illegal Goods", "Child Safety", "Sexual Abuse", "Profanity", "Self Harm/Suicide", "Violent Extremism","None"]360 # Here, you might want to process combined_analysis_results to summarize or just display them361 362 # display_categories(" ".join(categories_results), categories)363 364 # Extract audio365 audio_command = [366 'ffmpeg',367 '-i', stream_url, # Input stream URL368 '-vn', # Ignore the video for the audio output369 '-acodec', 'libmp3lame', # Set the audio codec to MP3370 '-t', str(seconds), # Duration for the audio extraction (selected seconds)371 '-f', 'mp3', # Output format as MP3372 '-'373 ]374 audio_output, _ = execute_ffmpeg_command(audio_command)375 376 st.write("Extracted Audio:")377 audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")378 audio_tempfile.write(audio_output)379 audio_tempfile.close()380 381 st.audio(audio_output, format='audio/mpeg', start_time=0)382 383 # Get the transcript from whisper384 transcript_text = get_transcript_from_audio(audio_tempfile.name)385 if transcript_text:386 st.markdown("**Transcript:**")387 st.write(transcript_text)388 else:389 st.write("Failed to retrieve transcript.")390 391 392 # Get consolidated description for all frames393 if ffmpeg_output:394 description = generate_description(base64_frames,prompt)395 if description:396 st.markdown("**Frame Description:**")397 st.write(description)398 else:399 st.write("Failed to generate description.")400 401 image_labels = analyze_image_with_google_vision_api(frame_base64)402 # st.write(image_labels)403 analysis_result = analyze_content_with_openai(extracted_text, description, image_labels)404 st.write(analysis_result)405 display_categories(analysis_result, categories)406 categories_results.append(analysis_result) # Collect results for summary407 408 # Get the transcript from whisper409 transcript_text = get_transcript_from_audio(audio_tempfile.name) 410 description = generate_description(base64_frames,prompt)411 # Generate overall description using transcript and video description412 overall_description = generate_overall_description(transcript_text, description)413 if overall_description:414 st.markdown("**Consolidated Description:**")415 st.write(overall_description)416 else:417 st.write("Failed to generate overall description.")418 419 elif uploaded_video is not None and extract_frames_button:420 with tempfile.NamedTemporaryFile(delete=False, suffix=".mp4") as tmpfile:421 tmpfile.write(uploaded_video.getvalue())422 video_file_path = tmpfile.name423 424 ffmpeg_command = [425 'ffmpeg', # Input stream URL426 '-i', video_file_path, 427 '-t', str(seconds), # Duration to process the input (selected seconds)428 '-vf', 'fps=1', # Extract one frame per second429 '-f', 'image2pipe', # Output format as image2pipe430 '-c:v', 'mjpeg', # Codec for output video431 '-an', # No audio432 '-'433 ]434 435 ffmpeg_output = execute_fmpeg_command(ffmpeg_command)436 437 if ffmpeg_output:438 st.write("Frames Extracted:")439 frame_bytes_list = ffmpeg_output.split(b'\xff\xd8')[1:] # Correct splitting for JPEG frames440 n_frames = len(frame_bytes_list)441 base64_frames = [base64.b64encode(b'\xff\xd8' + frame_bytes).decode('utf-8') for frame_bytes in frame_bytes_list]442 443 frame_dict = {}444 categories_results = []445 frame_texts = {}446 447 for idx, frame_base64 in enumerate(base64_frames):448 extracted_text = extract_text_from_base64_frame(frame_base64)449 frame_texts[idx] = extracted_text450 # Use Streamlit columns for side-by-side display (1 column for image, 1 for text)451 col1, col2 = st.columns([3, 2])452 with col1:453 frame_bytes = base64.b64decode(frame_base64)454 frame_dict[idx + 1] = frame_bytes455 st.image(Image.open(BytesIO(frame_bytes)), caption=f'Frame {idx + 1}', use_column_width=True)456 with col2:457 st.write(f"Extracted Text: {extracted_text}")458 459 460 st.write("Analysis Results for All Frames:")461 # Assuming 'categories' is defined with all possible categories you're interested in462 categories = ["Bullying", "Nudity & Adult Content", "Graphic Violence", "Illegal Goods", "Child Safety", "Sexual Abuse", "Profanity", "Self Harm/Suicide", "Violent Extremism", "None"]463 # Here, you might want to process combined_analysis_results to summarize or just display them464 465 466 467 468 # Extract audio469 audio_command = [470 'ffmpeg',471 '-i', video_file_path, 472 '-t', str(seconds), 473 '-vf', 'fps=1', # Input stream URL474 '-vn', # Ignore the video for the audio output475 '-acodec', 'libmp3lame', # Set the audio codec to MP3 # Duration for the audio extraction (selected seconds)476 '-f', 'mp3', # Output format as MP3477 '-'478 ]479 audio_output, _ = execute_ffmpeg_command(audio_command)480 481 st.write("Extracted Audio:")482 audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")483 audio_tempfile.write(audio_output)484 audio_tempfile.close()485 486 st.audio(audio_output, format='audio/mpeg', start_time=0)487 488 # Get the transcript from whisper489 transcript_text = get_transcript_from_audio(audio_tempfile.name)490 if transcript_text:491 st.markdown("**Transcript:**")492 st.write(transcript_text)493 else:494 st.write("Failed to retrieve transcript.")495 496 # Get consolidated description for all frames497 if ffmpeg_output:498 description,frame_numbers = generate_description(base64_frames,prompt)499 if description:500 st.markdown("**Frame Description:**")501 st.write(description)502 else:503 st.write("Failed to generate description.")504 505 image_labels = analyze_image_with_google_vision_api(frame_base64)506 # st.write(image_labels)507 analysis_result = analyze_content_with_openai(extracted_text, description, image_labels)508 st.write(analysis_result)509 display_categories(analysis_result, categories)510 categories_results.append(analysis_result) # Collect results for summary511 512 # if st.button("Overall Description"): 513 # audio_tempfile = tempfile.NamedTemporaryFile(delete=False, suffix=".mp3")514 # audio_tempfile.write(audio_output)515 # audio_tempfile.close()516 517 # Get the transcript from whisper518 transcript_text = get_transcript_from_audio(audio_tempfile.name) 519 description = generate_description(base64_frames,prompt)520 521 if frame_numbers:522 print("Frame numbers to extract:", frame_numbers) # Check frame numbers523 524 # Create a mapping from original frame numbers to sequential numbers525 frame_mapping = {}526 new_frame_numbers = []527 for idx, frame_number in enumerate(sorted(frame_numbers)):528 frame_mapping[frame_number] = idx + 1529 new_frame_numbers.append(idx + 1)530 531 print("New frame numbers:", new_frame_numbers)532 print("Frame mapping:", frame_mapping)533 534 # Create a temporary directory to store images535 with tempfile.TemporaryDirectory() as temp_dir:536 image_paths = []537 for frame_number in frame_numbers:538 if frame_number in frame_dict:539 frame_path = os.path.join(temp_dir, f'frame_{frame_mapping[frame_number]:03}.jpg') # Updated file naming540 image_paths.append(frame_path)541 with open(frame_path, 'wb') as f:542 f.write(frame_dict[frame_number])543 544 #image = Image.open(BytesIO(frame_bytes))545 #st.image(image, caption='Selected Frame', use_column_width=True)546 #with open(frame_path, "rb") as file:547 # btn = st.download_button(548 # label="Download Frame",549 # data=file,550 # file_name=f'frame_{frame_number}.jpg',551 # mime="image/jpeg"552 # )553 # Once all selected frames are saved as images, create a video from them using FFmpeg554 video_output_path = os.path.join(temp_dir, 'output7.mp4')555 framerate = 1 # Adjust framerate based on the number of frames556 ffmpeg_command = [557 'ffmpeg',558 '-framerate', str(framerate), # Set framerate based on the number of frames559 '-i', os.path.join(temp_dir, 'frame_%03d.jpg'), # Input pattern for all frame files560 '-c:v', 'libx264',561 '-pix_fmt', 'yuv420p',562 video_output_path563 ]564 565 print("FFmpeg command:", ' '.join(ffmpeg_command)) # Debug FFmpeg command566 567 subprocess.run(ffmpeg_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)568 569 # Display or provide a download link for the created video570 st.header("Final Video")571 st.video(video_output_path)572 # Generate overall description using transcript and video description573 overall_description = generate_overall_description(transcript_text, description)574 if overall_description:575 st.markdown("**Consolidated Description:**")576 st.write(overall_description)577 else:578 st.write("Failed to generate overall description.")579 580 else:581 st.write(" ")