Poojith28/orbit
0
1import streamlit as st2import pandas as pd3import numpy as np4import plotly.express as px5import plotly.graph_objects as go6from plotly.subplots import make_subplots7import matplotlib.pyplot as plt8from wordcloud import WordCloud, STOPWORDS9from sklearn.feature_extraction.text import CountVectorizer10from sklearn.decomposition import LatentDirichletAllocation11import re12import networkx as nx13from datetime import datetime, timedelta14import seaborn as sns15from scipy import stats16import warnings17warnings.filterwarnings('ignore')18 19# -----------------------------------------------------------------------------20# 1. Page configuration & styling21# -----------------------------------------------------------------------------22st.set_page_config(23 page_title='Social Toxicity Analytics Platform',24 layout='wide',25 initial_sidebar_state='expanded',26 page_icon='📊'27)28 29# Custom CSS for professional styling30st.markdown("""31<style>32 .main-header {33 background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);34 padding: 2rem;35 border-radius: 10px;36 margin-bottom: 2rem;37 color: white;38 text-align: center;39 }40 .metric-card {41 background: white;42 padding: 1.5rem;43 border-radius: 8px;44 box-shadow: 0 2px 4px rgba(0,0,0,0.1);45 border-left: 4px solid #667eea;46 }47 .sidebar .sidebar-content {48 background: linear-gradient(180deg, #f8f9fa 0%, #e9ecef 100%);49 }50 .stAlert {51 border-radius: 8px;52 }53 .section-header {54 background: #f8f9fa;55 padding: 1rem;56 border-radius: 8px;57 margin: 1rem 0;58 border-left: 4px solid #667eea;59 }60</style>61""", unsafe_allow_html=True)62 63# Header64st.markdown("""65<div class="main-header">66 <h1>🛡️ Social Toxicity Analytics Platform</h1>67 <p>Advanced insights into online discourse patterns and toxicity trends</p>68</div>69""", unsafe_allow_html=True)70 71# -----------------------------------------------------------------------------72# 2. Enhanced sidebar controls73# -----------------------------------------------------------------------------74st.sidebar.markdown("### 📁 Data Configuration")75DATA_PATH = st.sidebar.text_input(76 'CSV data path',77 value='full_call_no_null_columns.csv',78 help='Path to your consolidated CSV file'79)80 81st.sidebar.markdown("### 📅 Time Range Filters")82col1, col2 = st.sidebar.columns(2)83with col1:84 min_date = st.date_input('Start Date', None)85with col2:86 max_date = st.date_input('End Date', None)87 88st.sidebar.markdown("### 🎯 Analysis Parameters")89tox_threshold = st.sidebar.slider(90 'Toxicity Threshold',91 0.0, 1.0, 0.5, 0.01,92 help='Minimum toxicity score for filtering'93)94 95min_comments = st.sidebar.slider(96 'Minimum Comments per Video',97 1, 100, 5,98 help='Filter videos with fewer comments'99)100 101st.sidebar.markdown("### 🧭 Navigation")102page = st.sidebar.radio(103 'Select Analysis View',104 ['📊 Executive Dashboard', '🔍 Toxicity Deep Dive', '📈 Trend Analysis', 105 '⏱️ User Timeline', '🗝️ Content Analysis', '🌐 Network Insights']106)107 108# -----------------------------------------------------------------------------109# 3. Enhanced data loading & preprocessing110# -----------------------------------------------------------------------------111@st.cache_data(show_spinner='Loading and processing data...')112def load_and_prepare(path):113 try:114 df = pd.read_csv(path)115 116 # Parse timestamps with better error handling117 if 'create_time_x' in df.columns:118 df['comment_time'] = pd.to_datetime(119 df['create_time_x'], unit='s', errors='coerce'120 )121 if 'createTime' in df.columns:122 df['video_time'] = pd.to_datetime(123 df['createTime'], unit='s', errors='coerce'124 )125 126 # Handle comment text127 if 'desc_x' in df.columns and df['desc_x'].notna().sum() > 0:128 df['comment'] = df['desc_x'].astype(str)129 elif 'text_x' in df.columns:130 df['comment'] = df['text_x'].astype(str)131 else:132 df['comment'] = ''133 134 # Create comprehensive toxicity flags135 tox_cols = ['toxicity_label', 'severe_toxicity_label', 'obscene_label',136 'threat_label', 'insult_label', 'identity_attack_label']137 present_cols = [c for c in tox_cols if c in df.columns]138 139 if 'any_toxic' not in df.columns and present_cols:140 df['any_toxic'] = df[present_cols].any(axis=1).astype(int)141 elif 'any_toxic' not in df.columns:142 df['any_toxic'] = 0143 144 # Add derived features145 df['comment_length'] = df['comment'].str.len()146 df['word_count'] = df['comment'].str.split().str.len()147 df['hour'] = df['comment_time'].dt.hour148 df['day_of_week'] = df['comment_time'].dt.day_name()149 150 return df151 except Exception as e:152 st.error(f"Error loading data: {str(e)}")153 return pd.DataFrame()154 155# Load data156df = load_and_prepare(DATA_PATH)157 158if df.empty:159 st.error("No data loaded. Please check your file path.")160 st.stop()161 162# Apply filters163if min_date:164 df = df[df['comment_time'] >= pd.to_datetime(min_date)]165if max_date:166 df = df[df['comment_time'] <= pd.to_datetime(max_date)]167 168# -----------------------------------------------------------------------------169# 4. Enhanced helper functions170# -----------------------------------------------------------------------------171def create_metric_card(title, value, delta=None, delta_color="normal"):172 """Create a styled metric display"""173 return st.metric(174 label=title,175 value=value,176 delta=delta,177 delta_color=delta_color178 )179 180def advanced_word_cloud(text_series, max_words=100, colormap='viridis'):181 """Generate an advanced word cloud"""182 stop_words = STOPWORDS.union({'user', 'video', 'comment', 'like', 'get'})183 text = ' '.join(text_series.astype(str).tolist())184 185 if len(text.strip()) == 0:186 st.warning("No text data available for word cloud")187 return188 189 wc = WordCloud(190 width=1200, 191 height=600,192 stopwords=stop_words,193 max_words=max_words,194 background_color='white',195 colormap=colormap,196 collocations=False197 ).generate(text)198 199 fig, ax = plt.subplots(figsize=(15, 8))200 ax.imshow(wc, interpolation='bilinear')201 ax.axis('off')202 st.pyplot(fig)203 204def sentiment_analysis(text_series, n_topics=5):205 """Perform topic modeling with LDA"""206 if len(text_series) == 0:207 return []208 209 # Clean text210 text_clean = text_series.astype(str).str.lower()211 text_clean = text_clean.str.replace(r'[^\w\s]', '', regex=True)212 213 try:214 vectorizer = CountVectorizer(215 max_features=1000,216 stop_words='english',217 min_df=2,218 max_df=0.95219 )220 X = vectorizer.fit_transform(text_clean)221 222 lda = LatentDirichletAllocation(223 n_components=n_topics,224 random_state=42,225 max_iter=10226 )227 lda.fit(X)228 229 vocab = vectorizer.get_feature_names_out()230 topics = []231 232 for i, comp in enumerate(lda.components_):233 top_words_idx = comp.argsort()[-10:][::-1]234 top_words = [vocab[idx] for idx in top_words_idx]235 topics.append({236 'topic': i+1,237 'words': ', '.join(top_words),238 'weight': comp[top_words_idx[0]]239 })240 241 return topics242 except Exception as e:243 st.error(f"Topic analysis failed: {str(e)}")244 return []245 246# -----------------------------------------------------------------------------247# 5. Page implementations248# -----------------------------------------------------------------------------249 250if page == '📊 Executive Dashboard':251 st.markdown('<div class="section-header"><h2>📊 Executive Summary</h2></div>', unsafe_allow_html=True)252 253 # Key Performance Indicators254 col1, col2, col3, col4, col5 = st.columns(5)255 256 total_comments = len(df)257 unique_commenters = df['unique_id_x'].nunique() if 'unique_id_x' in df.columns else 0258 unique_videos = df['aweme_id_x'].nunique() if 'aweme_id_x' in df.columns else 0259 toxic_rate = df['any_toxic'].mean() * 100 if 'any_toxic' in df.columns else 0260 avg_engagement = df.groupby('aweme_id_x').size().mean() if 'aweme_id_x' in df.columns else 0261 262 with col1:263 create_metric_card("Total Comments", f"{total_comments:,}")264 with col2:265 create_metric_card("Unique Users", f"{unique_commenters:,}")266 with col3:267 create_metric_card("Videos Analyzed", f"{unique_videos:,}")268 with col4:269 create_metric_card("Toxicity Rate", f"{toxic_rate:.1f}%")270 with col5:271 create_metric_card("Avg Engagement", f"{avg_engagement:.1f}")272 273 # Dashboard charts274 col1, col2 = st.columns(2)275 276 with col1:277 st.subheader("📈 Toxicity Distribution")278 if 'toxicity' in df.columns:279 fig = px.histogram(280 df, 281 x='toxicity',282 nbins=50,283 title='Toxicity Score Distribution',284 color_discrete_sequence=['#667eea']285 )286 fig.update_layout(287 showlegend=False,288 plot_bgcolor='rgba(0,0,0,0)',289 paper_bgcolor='rgba(0,0,0,0)'290 )291 st.plotly_chart(fig, use_container_width=True)292 293 with col2:294 st.subheader("📊 Toxicity by Type")295 if 'toxicity' in df.columns:296 tox_types = ['severe_toxicity', 'obscene', 'threat', 'insult', 'identity_attack']297 available_types = [t for t in tox_types if t in df.columns]298 299 if available_types:300 type_means = df[available_types].mean()301 fig = px.bar(302 x=available_types,303 y=type_means.values,304 title='Average Toxicity by Type',305 color=type_means.values,306 color_continuous_scale='Reds'307 )308 fig.update_layout(309 showlegend=False,310 plot_bgcolor='rgba(0,0,0,0)',311 paper_bgcolor='rgba(0,0,0,0)'312 )313 st.plotly_chart(fig, use_container_width=True)314 315 # Time series analysis316 st.subheader("📅 Activity Timeline")317 if 'comment_time' in df.columns:318 df['date'] = df['comment_time'].dt.date319 daily_stats = df.groupby('date').agg({320 'comment': 'count',321 'toxicity': 'mean' if 'toxicity' in df.columns else lambda x: 0,322 'any_toxic': 'sum' if 'any_toxic' in df.columns else lambda x: 0323 }).reset_index()324 325 fig = make_subplots(326 rows=2, cols=1,327 subplot_titles=('Daily Comment Volume', 'Daily Toxicity Rate'),328 vertical_spacing=0.1329 )330 331 fig.add_trace(332 go.Scatter(333 x=daily_stats['date'],334 y=daily_stats['comment'],335 mode='lines+markers',336 name='Comments',337 line=dict(color='#667eea', width=2)338 ),339 row=1, col=1340 )341 342 fig.add_trace(343 go.Scatter(344 x=daily_stats['date'],345 y=daily_stats['toxicity'],346 mode='lines+markers',347 name='Avg Toxicity',348 line=dict(color='#e74c3c', width=2)349 ),350 row=2, col=1351 )352 353 fig.update_layout(354 height=500,355 showlegend=False,356 plot_bgcolor='rgba(0,0,0,0)',357 paper_bgcolor='rgba(0,0,0,0)'358 )359 st.plotly_chart(fig, use_container_width=True)360 361 # Top toxic content362 st.subheader("🚨 High-Risk Content")363 if 'toxicity' in df.columns:364 toxic_content = df.nlargest(10, 'toxicity')[365 ['comment_time', 'comment', 'toxicity', 'unique_id_x']366 ]367 st.dataframe(toxic_content, use_container_width=True)368 369elif page == '🔍 Toxicity Deep Dive':370 st.markdown('<div class="section-header"><h2>🔍 Advanced Toxicity Analysis</h2></div>', unsafe_allow_html=True)371 372 # Correlation analysis373 st.subheader("📊 Toxicity Correlations")374 if 'toxicity' in df.columns:375 tox_cols = ['toxicity', 'severe_toxicity', 'obscene', 'threat', 'insult', 'identity_attack']376 available_cols = [c for c in tox_cols if c in df.columns]377 378 if len(available_cols) > 1:379 corr_matrix = df[available_cols].corr()380 381 fig = px.imshow(382 corr_matrix,383 text_auto=True,384 aspect="auto",385 color_continuous_scale='RdBu',386 title="Toxicity Metrics Correlation Matrix"387 )388 st.plotly_chart(fig, use_container_width=True)389 390 # Scatter plot analysis391 col1, col2 = st.columns(2)392 393 with col1:394 st.subheader("📈 Comment Length vs Toxicity")395 if 'toxicity' in df.columns and 'comment_length' in df.columns:396 fig = px.scatter(397 df.sample(min(1000, len(df))),398 x='comment_length',399 y='toxicity',400 opacity=0.6,401 trendline="ols",402 title="Comment Length vs Toxicity Score"403 )404 st.plotly_chart(fig, use_container_width=True)405 406 with col2:407 st.subheader("🕐 Toxicity by Hour")408 if 'toxicity' in df.columns and 'hour' in df.columns:409 hourly_tox = df.groupby('hour')['toxicity'].mean().reset_index()410 fig = px.bar(411 hourly_tox,412 x='hour',413 y='toxicity',414 title="Average Toxicity by Hour of Day"415 )416 st.plotly_chart(fig, use_container_width=True)417 418 # User behavior analysis419 st.subheader("👥 User Behavior Patterns")420 if 'unique_id_x' in df.columns and 'toxicity' in df.columns:421 user_stats = df.groupby('unique_id_x').agg({422 'comment': 'count',423 'toxicity': 'mean',424 'any_toxic': 'sum'425 }).reset_index()426 user_stats.columns = ['user_id', 'total_comments', 'avg_toxicity', 'toxic_comments']427 user_stats['toxicity_rate'] = user_stats['toxic_comments'] / user_stats['total_comments']428 429 fig = px.scatter(430 user_stats[user_stats['total_comments'] >= 5],431 x='total_comments',432 y='avg_toxicity',433 size='toxic_comments',434 hover_data=['toxicity_rate'],435 title="User Activity vs Average Toxicity",436 labels={'total_comments': 'Total Comments', 'avg_toxicity': 'Average Toxicity'}437 )438 st.plotly_chart(fig, use_container_width=True)439 440elif page == '⏱️ User Timeline':441 st.markdown('<div class="section-header"><h2>⏱️ Interactive User Timeline</h2></div>', unsafe_allow_html=True)442 443 # Tab selection for different timeline views444 tab1, tab2 = st.tabs(["📊 Original Timeline Dashboard", "🔍 Multi-User Timeline"])445 446 with tab1:447 # Your original timeline code implementation448 st.subheader("🎬 Video Timeline & Commenters Dashboard")449 450 # — Compute total comments per video451 comment_counts = (452 df.groupby("aweme_id_x")453 .size()454 .rename("total_comments")455 .reset_index()456 )457 458 # — Build a videos DataFrame with one row per video459 videos = (460 df[["aweme_id_x", "uniqueId", "createTime"]]461 .drop_duplicates("aweme_id_x")462 .merge(comment_counts, on="aweme_id_x")463 )464 465 # — Sidebar controls for original timeline466 col1, col2 = st.columns(2)467 468 with col1:469 scale = st.slider("Bubble size scale", 0.1, 5.0, 0.5, 0.1, key="bubble_scale")470 471 with col2:472 # — Power Commenters filters473 st.write("**Power Commenters Filter**")474 min_comments = st.number_input("Min total comments", min_value=1, value=2, step=1, key="min_comments")475 min_authors = st.number_input("Min distinct authors", min_value=1, value=2, step=1, key="min_authors")476 477 # — Compute commenter stats478 stats = (479 df.groupby("unique_id_x")480 .agg(481 total_comments = ("aweme_id_x", "count"),482 distinct_authors = ("uniqueId", "nunique"),483 )484 .reset_index()485 )486 power_users = stats[487 (stats.total_comments >= min_comments) &488 (stats.distinct_authors >= min_authors)489 ].sort_values(["total_comments","distinct_authors"], ascending=False)490 491 st.subheader("Power Commenters")492 st.write(493 f"Commenters with ≥ {min_comments} comments "494 f"across ≥ {min_authors} distinct authors:"495 )496 st.dataframe(power_users, use_container_width=True)497 498 # — Identify the main video authors499 video_authors = sorted(videos["uniqueId"].unique())500 501 # — Commenter selection502 all_commenters = sorted(df["unique_id_x"].unique())503 selected_commenter = st.selectbox(504 "Highlight a commentator on the timeline (or 'None')",505 ["None"] + all_commenters,506 key="selected_commenter"507 )508 509 # — Build the y-axis categories510 y_categories = video_authors.copy()511 if selected_commenter != "None" and selected_commenter not in y_categories:512 y_categories.append(selected_commenter)513 user_pos = {uid: idx for idx, uid in enumerate(y_categories)}514 515 # — Create the Plotly figure516 fig = go.Figure()517 518 # 1) Plot video bubbles sized by total_comments - WITH ERROR FIX519 for author in video_authors:520 vids = videos[videos["uniqueId"] == author]521 522 # Create text with proper timestamp handling523 text_labels = []524 for ts, cnt in zip(vids["createTime"], vids["total_comments"]):525 try:526 # Check if timestamp is valid527 if pd.isna(ts) or ts is pd.NaT:528 formatted_time = "Invalid Date"529 else:530 formatted_time = ts.strftime('%Y-%m-%d %H:%M')531 532 text_labels.append(533 f"{author}<br>{formatted_time}<br>"534 f"Comments: {cnt}"535 )536 except (AttributeError, ValueError):537 # Handle any other timestamp formatting errors538 text_labels.append(539 f"{author}<br>Invalid Date<br>"540 f"Comments: {cnt}"541 )542 543 fig.add_trace(go.Scatter(544 x=vids["createTime"],545 y=[user_pos[author]] * len(vids),546 mode="markers",547 marker=dict(548 size=vids["total_comments"] * scale,549 opacity=0.7550 ),551 name=f"{author} (videos)",552 text=text_labels,553 hoverinfo="text",554 ))555 556 # 2) Draw internal edges (author↔author comments)557 internal = df[558 df["unique_id_x"].isin(video_authors) &559 df["uniqueId"].isin(video_authors) &560 (df["unique_id_x"] != df["uniqueId"])561 ]562 for _, row in internal.iterrows():563 t = row["create_time_x"]564 y0 = user_pos[row["unique_id_x"]]565 y1 = user_pos[row["uniqueId"]]566 fig.add_trace(go.Scatter(567 x=[t, t],568 y=[y0, y1],569 mode="lines",570 line=dict(width=1, dash="dot", color="gray"),571 hoverinfo="none",572 showlegend=False,573 ))574 575 # 3) Draw edges for the selected external commentator576 if selected_commenter != "None":577 comments = df[df["unique_id_x"] == selected_commenter]578 for _, row in comments.iterrows():579 t = row["create_time_x"]580 y0 = user_pos[selected_commenter]581 y1 = user_pos[row["uniqueId"]]582 fig.add_trace(go.Scatter(583 x=[t, t],584 y=[y0, y1],585 mode="lines",586 line=dict(width=1, dash="solid", color="blue"),587 hoverinfo="none",588 showlegend=False,589 ))590 591 # 4) Final layout tweaks592 fig.update_layout(593 title="Video Authors Timeline with Comment‐Count‐Sized Bubbles",594 xaxis_title="Time",595 yaxis=dict(596 tickmode="array",597 tickvals=list(user_pos.values()),598 ticktext=list(user_pos.keys()),599 title="Account",600 ),601 height=600,602 margin=dict(l=50, r=50, t=60, b=50),603 plot_bgcolor='rgba(0,0,0,0)',604 paper_bgcolor='rgba(0,0,0,0)'605 )606 607 # Render in Streamlit608 st.plotly_chart(fig, use_container_width=True)609 610 # — Display the highlighted commentator's comments & toxicity611 if selected_commenter != "None":612 st.subheader(f"Comments by {selected_commenter}")613 commenter_df = df[df["unique_id_x"] == selected_commenter][614 ["create_time_x", "text_x", "toxicity"]615 ].sort_values("create_time_x")616 commenter_df = commenter_df.rename(columns={617 "create_time_x": "Timestamp",618 "text_x": "Comment",619 "toxicity": "Toxicity"620 })621 st.dataframe(commenter_df, use_container_width=True)622 623 with tab2:624 # Multi-User Timeline from the comprehensive version625 st.subheader("👤 Multi-User Selection")626 627 col1, col2 = st.columns([2, 1])628 629 with col1:630 if 'unique_id_x' in df.columns:631 # Get user activity stats632 user_activity = df.groupby('unique_id_x').agg({633 'text_x': 'count', # Using text_x instead of 'comment'634 'aweme_id_x': 'nunique',635 'toxicity': 'mean' if 'toxicity' in df.columns else lambda x: 0636 }).reset_index()637 user_activity.columns = ['user_id', 'total_comments', 'videos_engaged', 'avg_toxicity']638 user_activity = user_activity.sort_values('total_comments', ascending=False)639 640 # Multi-select for users641 selected_users = st.multiselect(642 "Select users to analyze (top active users shown first):",643 options=user_activity['user_id'].head(50).tolist(),644 default=user_activity['user_id'].head(5).tolist(),645 help="Select multiple users to compare their activity patterns",646 key="multi_user_select"647 )648 649 with col2:650 st.subheader("📊 Selection Stats")651 if selected_users:652 for user in selected_users:653 user_data = user_activity[user_activity['user_id'] == user].iloc[0]654 st.write(f"**User {user}:**")655 st.write(f"- Comments: {user_data['total_comments']}")656 st.write(f"- Videos: {user_data['videos_engaged']}")657 st.write(f"- Avg Toxicity: {user_data['avg_toxicity']:.3f}")658 st.write("---")659 660 # Multi-user timeline visualization661 if selected_users:662 st.subheader("📈 Multi-User Activity Timeline")663 664 # Prepare timeline data665 timeline_data = []666 video_data = []667 668 for user in selected_users:669 user_comments = df[df['unique_id_x'] == user].copy()670 671 # Get video posting times672 user_videos = user_comments.groupby('aweme_id_x').agg({673 'create_time_x': 'min', # First comment time as proxy674 'text_x': 'count'675 }).reset_index()676 user_videos.columns = ['video_id', 'post_time', 'total_comments']677 678 # Add video bubbles679 for _, video in user_videos.iterrows():680 video_data.append({681 'user': user,682 'video_id': video['video_id'],683 'post_time': video['post_time'],684 'total_comments': video['total_comments'],685 'type': 'video'686 })687 688 # Add comment connections689 for _, comment in user_comments.iterrows():690 timeline_data.append({691 'user': user,692 'video_id': comment['aweme_id_x'],693 'comment_time': comment['create_time_x'],694 'toxicity': float(comment.get('toxicity', 0)),695 'type': 'comment'696 })697 698 # Create multi-user timeline plot699 fig2 = go.Figure()700 701 import plotly.express as px702 colors = px.colors.qualitative.Set3[:len(selected_users)]703 704 # Add video bubbles for each user - WITH ERROR FIX705 for i, user in enumerate(selected_users):706 user_videos = [v for v in video_data if v['user'] == user]707 708 if user_videos:709 # Create safe text labels for multi-user timeline710 text_labels_multi = []711 for v in user_videos:712 try:713 if pd.isna(v['post_time']) or v['post_time'] is pd.NaT:714 formatted_time = "Invalid Date"715 else:716 formatted_time = v['post_time'].strftime('%Y-%m-%d %H:%M')717 718 text_labels_multi.append(f"Video: {v['video_id']}<br>Time: {formatted_time}<br>Comments: {v['total_comments']}")719 except (AttributeError, ValueError):720 text_labels_multi.append(f"Video: {v['video_id']}<br>Time: Invalid Date<br>Comments: {v['total_comments']}")721 722 fig2.add_trace(go.Scatter(723 x=[v['post_time'] for v in user_videos],724 y=[i] * len(user_videos),725 mode='markers',726 marker=dict(727 size=[min(v['total_comments'] * 5, 100) for v in user_videos],728 color=colors[i],729 opacity=0.7,730 line=dict(width=2, color='darkblue')731 ),732 name=f'User {user} Videos',733 text=text_labels_multi,734 hovertemplate='<b>%{text}</b><br>Posted: %{x}<extra></extra>'735 ))736 737 # Add comment connections between users738 for i, user in enumerate(selected_users):739 user_comments = [c for c in timeline_data if c['user'] == user]740 741 # Group comments by video742 video_comments = {}743 for comment in user_comments:744 video_id = comment['video_id']745 if video_id not in video_comments:746 video_comments[video_id] = []747 video_comments[video_id].append(comment)748 749 # Create connections to other users' videos750 for video_id, comments in video_comments.items():751 # Find if this video belongs to another user in selection752 video_owner = None753 owner_index = None754 for j, other_user in enumerate(selected_users):755 if any(v['video_id'] == video_id and v['user'] == other_user for v in video_data):756 video_owner = other_user757 owner_index = j758 break759 760 if video_owner and video_owner != user:761 # Get video post time762 video_post_time = next(v['post_time'] for v in video_data if v['video_id'] == video_id and v['user'] == video_owner)763 764 # Draw connection line765 fig2.add_trace(go.Scatter(766 x=[video_post_time, video_post_time],767 y=[owner_index, i],768 mode='lines',769 line=dict(770 width=min(len(comments) * 2, 10),771 color='red',772 dash='dash'773 ),774 opacity=0.6,775 showlegend=False,776 hovertemplate=f'User {user} → User {video_owner}<br>Comments: {len(comments)}<extra></extra>'777 ))778 779 # Update layout780 fig2.update_layout(781 title="Multi-User Video Interaction Timeline",782 xaxis_title="Time",783 yaxis_title="Users",784 yaxis=dict(785 tickmode='array',786 tickvals=list(range(len(selected_users))),787 ticktext=[f'User {user}' for user in selected_users]788 ),789 height=max(400, len(selected_users) * 80),790 hovermode='closest',791 plot_bgcolor='rgba(0,0,0,0)',792 paper_bgcolor='rgba(0,0,0,0)'793 )794 795 st.plotly_chart(fig2, use_container_width=True)796 797 # Legend and explanation798 st.info("""799 **Multi-User Timeline Legend:**800 - 🔵 **Bubbles**: Videos posted by users (size = total comments received)801 - 📏 **Dashed Lines**: Comments from other users (thickness = number of comments)802 - 🎨 **Colors**: Each user has a unique color803 """)804 805 # Interaction statistics806 st.subheader("📊 Cross-User Interactions")807 808 interaction_matrix = pd.DataFrame(0, index=selected_users, columns=selected_users)809 810 for user in selected_users:811 user_comments = df[df['unique_id_x'] == user]812 for _, comment in user_comments.iterrows():813 video_id = comment['aweme_id_x']814 # Find video owner815 for other_user in selected_users:816 if other_user != user:817 other_user_videos = df[df['unique_id_x'] == other_user]['aweme_id_x'].unique()818 if video_id in other_user_videos:819 interaction_matrix.loc[user, other_user] += 1820 821 if interaction_matrix.sum().sum() > 0:822 import plotly.express as px823 fig3 = px.imshow(824 interaction_matrix,825 text_auto=True,826 aspect="auto",827 color_continuous_scale='Blues',828 title="User Interaction Matrix (Comments on Each Other's Videos)"829 )830 st.plotly_chart(fig3, use_container_width=True)831 832 else:833 st.info("Please select users to analyze their timeline interactions.")834 835elif page == '📈 Trend Analysis':836 st.markdown('<div class="section-header"><h2>📈 Comprehensive Trend Analysis</h2></div>', unsafe_allow_html=True)837 838 # Time-based analysis839 if 'comment_time' in df.columns:840 df['date'] = df['comment_time'].dt.date841 df['week'] = df['comment_time'].dt.isocalendar().week842 df['month'] = df['comment_time'].dt.month843 844 # Daily trends845 st.subheader("📅 Daily Patterns")846 daily_trends = df.groupby(['date', 'day_of_week']).agg({847 'comment': 'count',848 'toxicity': 'mean' if 'toxicity' in df.columns else lambda x: 0,849 'any_toxic': 'sum' if 'any_toxic' in df.columns else lambda x: 0850 }).reset_index()851 852 fig = px.line(853 daily_trends,854 x='date',855 y='comment',856 color='day_of_week',857 title='Daily Comment Volume by Day of Week'858 )859 st.plotly_chart(fig, use_container_width=True)860 861 # Weekly aggregation862 st.subheader("📊 Weekly Trends")863 weekly_trends = df.groupby('week').agg({864 'comment': 'count',865 'toxicity': 'mean' if 'toxicity' in df.columns else lambda x: 0,866 'any_toxic': 'sum' if 'any_toxic' in df.columns else lambda x: 0867 }).reset_index()868 869 fig = make_subplots(870 rows=2, cols=1,871 subplot_titles=('Weekly Comment Volume', 'Weekly Toxicity Rate'),872 vertical_spacing=0.1873 )874 875 fig.add_trace(876 go.Bar(x=weekly_trends['week'], y=weekly_trends['comment'], name='Comments'),877 row=1, col=1878 )879 880 fig.add_trace(881 go.Scatter(x=weekly_trends['week'], y=weekly_trends['toxicity'], 882 mode='lines+markers', name='Avg Toxicity'),883 row=2, col=1884 )885 886 fig.update_layout(height=500, showlegend=False)887 st.plotly_chart(fig, use_container_width=True)888 889elif page == '🗝️ Content Analysis':890 st.markdown('<div class="section-header"><h2>🗝️ Advanced Content Analysis</h2></div>', unsafe_allow_html=True)891 892 # Content filtering options893 col1, col2 = st.columns(2)894 with col1:895 content_filter = st.selectbox(896 "Analyze content by:",897 ["All Comments", "Toxic Comments Only", "Non-Toxic Comments Only"]898 )899 900 with col2:901 word_count_filter = st.slider(902 "Minimum word count:",903 1, 20, 3904 )905 906 # Filter data based on selection907 filtered_df = df.copy()908 if content_filter == "Toxic Comments Only":909 filtered_df = df[df['any_toxic'] == 1]910 elif content_filter == "Non-Toxic Comments Only":911 filtered_df = df[df['any_toxic'] == 0]912 913 filtered_df = filtered_df[filtered_df['word_count'] >= word_count_filter]914 915 if len(filtered_df) == 0:916 st.warning("No data matches the selected filters.")917 else:918 # Word cloud919 st.subheader("☁️ Word Cloud Analysis")920 advanced_word_cloud(filtered_df['comment'])921 922 # Topic modeling923 st.subheader("🏷️ Topic Modeling")924 n_topics = st.slider("Number of topics:", 3, 10, 5)925 926 if len(filtered_df) >= 10:927 topics = sentiment_analysis(filtered_df['comment'], n_topics)928 929 if topics:930 for topic in topics:931 st.write(f"**Topic {topic['topic']}:** {topic['words']}")932 else:933 st.warning("Not enough data for topic modeling.")934 935 # Content statistics936 st.subheader("📊 Content Statistics")937 938 col1, col2, col3 = st.columns(3)939 940 with col1:941 avg_length = filtered_df['comment_length'].mean()942 st.metric("Average Length", f"{avg_length:.1f} chars")943 944 with col2:945 avg_words = filtered_df['word_count'].mean()946 st.metric("Average Words", f"{avg_words:.1f}")947 948 with col3:949 unique_words = len(set(' '.join(filtered_df['comment'].astype(str)).split()))950 st.metric("Unique Words", f"{unique_words:,}")951 952 # Length vs toxicity analysis953 if 'toxicity' in filtered_df.columns:954 st.subheader("📏 Comment Length vs Toxicity")955 956 # Bin comments by length957 filtered_df['length_bin'] = pd.cut(filtered_df['comment_length'], 958 bins=10, labels=False)959 length_toxicity = filtered_df.groupby('length_bin').agg({960 'toxicity': 'mean',961 'comment': 'count'962 }).reset_index()963 964 fig = px.bar(965 length_toxicity,966 x='length_bin',967 y='toxicity',968 title='Average Toxicity by Comment Length',969 labels={'length_bin': 'Length Bin', 'toxicity': 'Average Toxicity'}970 )971 st.plotly_chart(fig, use_container_width=True)972 973elif page == '🌐 Network Insights':974 st.markdown('<div class="section-header"><h2>🌐 Network Analysis & Insights</h2></div>', unsafe_allow_html=True)975 976 # Network configuration977 st.subheader("⚙️ Network Configuration")978 979 col1, col2, col3 = st.columns(3)980 981 with col1:982 min_connections = st.slider("Minimum connections:", 1, 20, 2)983 984 with col2:985 max_nodes = st.slider("Maximum nodes to display:", 50, 500, 200)986 987 with col3:988 layout_type = st.selectbox("Layout algorithm:", 989 ["spring", "circular", "kamada_kawai", "random"])990 991 # Build network992 st.subheader("🕸️ User-Video Network")993 994 if 'unique_id_x' in df.columns and 'aweme_id_x' in df.columns:995 # Create edge list996 edges = df.groupby(['unique_id_x', 'aweme_id_x']).agg({997 'comment': 'count',998 'toxicity': 'mean' if 'toxicity' in df.columns else lambda x: 0999 }).reset_index()1000 edges.columns = ['user', 'video', 'weight', 'avg_toxicity']1001 1002 # Filter by minimum connections1003 edges = edges[edges['weight'] >= min_connections]1004 1005 # Limit nodes1006 if len(edges) > max_nodes:1007 edges = edges.nlargest(max_nodes, 'weight')1008 1009 if len(edges) == 0:1010 st.warning("No connections meet the minimum threshold.")1011 else:1012 # Build NetworkX graph1013 G = nx.Graph()1014 1015 # Add nodes and edges1016 for _, row in edges.iterrows():1017 user_node = f"user_{row['user']}"1018 video_node = f"video_{row['video']}"1019 1020 G.add_node(user_node, type='user', toxicity=row['avg_toxicity'])1021 G.add_node(video_node, type='video', toxicity=row['avg_toxicity'])1022 G.add_edge(user_node, video_node, weight=row['weight'])1023 1024 # Calculate layout1025 if layout_type == "spring":1026 pos = nx.spring_layout(G, k=0.5, iterations=50)1027 elif layout_type == "circular":1028 pos = nx.circular_layout(G)1029 elif layout_type == "kamada_kawai":1030 pos = nx.kamada_kawai_layout(G)1031 else:1032 pos = nx.random_layout(G)1033 1034 # Create plotly traces1035 edge_x = []1036 edge_y = []1037 edge_weights = []1038 1039 for edge in G.edges(data=True):1040 x0, y0 = pos[edge[0]]1041 x1, y1 = pos[edge[1]]1042 edge_x.extend([x0, x1, None])1043 edge_y.extend([y0, y1, None])1044 edge_weights.append(edge[2]['weight'])1045 1046 # Node traces1047 user_x = []1048 user_y = []1049 user_text = []1050 user_toxicity = []1051 1052 video_x = []1053 video_y = []1054 video_text = []1055 video_toxicity = []1056 1057 for node in G.nodes(data=True):1058 x, y = pos[node[0]]1059 1060 if node[1]['type'] == 'user':1061 user_x.append(x)1062 user_y.append(y)1063 user_text.append(node[0])1064 user_toxicity.append(node[1]['toxicity'])1065 else:1066 video_x.append(x)1067 video_y.append(y)1068 video_text.append(node[0])1069 video_toxicity.append(node[1]['toxicity'])1070 1071 # Create figure1072 fig = go.Figure()1073 1074 # Add edges1075 fig.add_trace(go.Scatter(1076 x=edge_x, y=edge_y,1077 line=dict(width=0.5, color='#888'),1078 hoverinfo='none',1079 mode='lines',1080 showlegend=False1081 ))1082 1083 # Add user nodes1084 fig.add_trace(go.Scatter(1085 x=user_x, y=user_y,1086 mode='markers',1087 hoverinfo='text',1088 text=user_text,1089 marker=dict(1090 showscale=True,1091 colorscale='Viridis',1092 size=15,1093 color=user_toxicity,1094 colorbar=dict(title="Avg Toxicity"),1095 line_width=2,1096 symbol='circle'1097 ),1098 name='Users'1099 ))1100 1101 # Add video nodes1102 fig.add_trace(go.Scatter(1103 x=video_x, y=video_y,1104 mode='markers',1105 hoverinfo='text',1106 text=video_text,1107 marker=dict(1108 size=12,1109 color='lightblue',1110 line=dict(width=2, color='darkblue'),1111 symbol='square'1112 ),1113 name='Videos'1114 ))1115 1116 fig.update_layout(1117 title='User-Video Interaction Network',1118 showlegend=True,1119 hovermode='closest',1120 margin=dict(b=20, l=5, r=5, t=40),1121 annotations=[1122 dict(1123 text="Users (circles) connected to Videos (squares)<br>Edge thickness = comment frequency",1124 showarrow=False,1125 xref="paper", yref="paper",1126 x=0.005, y=-0.002,1127 xanchor='left', yanchor='bottom',1128 font=dict(size=12)1129 )1130 ],1131 xaxis=dict(showgrid=False, zeroline=False, showticklabels=False),1132 yaxis=dict(showgrid=False, zeroline=False, showticklabels=False),1133 plot_bgcolor='rgba(0,0,0,0)',1134 paper_bgcolor='rgba(0,0,0,0)'1135 )1136 1137 st.plotly_chart(fig, use_container_width=True)1138 1139 # Network statistics1140 st.subheader("📊 Network Statistics")1141 1142 col1, col2, col3, col4 = st.columns(4)1143 1144 with col1:1145 st.metric("Total Nodes", len(G.nodes()))1146 1147 with col2:1148 st.metric("Total Edges", len(G.edges()))1149 1150 with col3:1151 density = nx.density(G)1152 st.metric("Network Density", f"{density:.3f}")1153 1154 with col4:1155 if len(G.nodes()) > 0:1156 avg_degree = sum(dict(G.degree()).values()) / len(G.nodes())1157 st.metric("Average Degree", f"{avg_degree:.1f}")1158 1159 # Community detection1160 st.subheader("🏘️ Community Detection")1161 1162 try:1163 communities = nx.community.greedy_modularity_communities(G)1164 st.write(f"**Number of communities detected:** {len(communities)}")1165 1166 for i, community in enumerate(communities[:5]): # Show top 5 communities1167 users = [node for node in community if node.startswith('user_')]1168 videos = [node for node in community if node.startswith('video_')]1169 1170 st.write(f"**Community {i+1}:** {len(users)} users, {len(videos)} videos")1171 1172 if len(communities) > 5:1173 st.write("... and more communities")1174 break1175 1176 except Exception as e:1177 st.warning(f"Community detection failed: {str(e)}")1178 1179 # Top influencers1180 st.subheader("🌟 Top Influencers")1181 1182 # Calculate centrality measures1183 try:1184 degree_centrality = nx.degree_centrality(G)1185 betweenness_centrality = nx.betweenness_centrality(G)1186 closeness_centrality = nx.closeness_centrality(G)1187 1188 # Create centrality dataframe1189 centrality_df = pd.DataFrame({1190 'node': list(degree_centrality.keys()),1191 'degree': list(degree_centrality.values()),1192 'betweenness': list(betweenness_centrality.values()),1193 'closeness': list(closeness_centrality.values())1194 })1195 1196 # Filter for users only1197 user_centrality = centrality_df[centrality_df['node'].str.startswith('user_')]1198 user_centrality = user_centrality.sort_values('degree', ascending=False).head(10)1199 1200 st.dataframe(user_centrality, use_container_width=True)