sapyeboa/News_topic_analyzer
0
1 2import gradio as gr3import json4import re5from collections import Counter6 7print("="*50)8print("Loading Topic Analyzer...")9print("="*50)10 11# Load all JSON data12with open('stopwords.json') as f:13 STOPWORDS = set(json.load(f))14print("Stopwords:", len(STOPWORDS))15 16with open('dictionary.json') as f:17 data = json.load(f)18 WORD2ID = data['word2id']19print("Dictionary:", len(WORD2ID), "words")20 21with open('bigrams.json') as f:22 BIGRAMS = json.load(f)23print("Bigrams:", len(BIGRAMS), "phrases")24 25with open('lda_topics.json') as f:26 LDA = json.load(f)27print("LDA:", LDA['num_topics'], "topics")28 29with open('lsa_topics.json') as f:30 LSA = json.load(f)31print("LSA:", LSA['num_topics'], "topics")32 33with open('bertopic_topics.json') as f:34 BERTOPIC = json.load(f)35print("BERTopic:", len(BERTOPIC.get('topics', {})), "topics")36 37with open('topic_names.json') as f:38 TOPIC_NAMES = json.load(f)39 40print("="*50)41print("Ready!")42print("="*50)43 44def read_file(file):45 if not file:46 return ""47 try:48 path = file.name if hasattr(file, 'name') else str(file)49 if path.lower().endswith('.txt'):50 return open(path, 'r', encoding='utf-8', errors='ignore').read()51 elif path.lower().endswith('.pdf'):52 import PyPDF253 with open(path, 'rb') as f:54 return " ".join([p.extract_text() or "" for p in PyPDF2.PdfReader(f).pages])55 elif path.lower().endswith('.docx'):56 from docx import Document57 return " ".join([p.text for p in Document(path).paragraphs])58 return open(path, 'r', errors='ignore').read()59 except:60 return ""61 62def preprocess(text):63 # Clean text64 text = re.sub(r'http\S+', '', text)65 text = re.sub(r'[^a-zA-Z\s]', ' ', text.lower())66 67 # Tokenize and remove stopwords68 tokens = [w for w in text.split() if len(w) >= 3 and w not in STOPWORDS]69 70 # Apply bigrams71 if BIGRAMS:72 new_tokens = []73 i = 074 while i < len(tokens):75 if i < len(tokens) - 1:76 pair = tokens[i] + " " + tokens[i+1]77 if pair in BIGRAMS:78 new_tokens.append(BIGRAMS[pair])79 i += 280 continue81 new_tokens.append(tokens[i])82 i += 183 tokens = new_tokens84 85 return tokens86 87def analyze_lda(text, n=5):88 tokens = preprocess(text)89 if len(tokens) < 3:90 return None, tokens, "Text too short"91 92 # Count word frequencies93 freq = Counter(tokens)94 95 # Score each topic96 scores = {}97 matches_by_topic = {}98 99 for tid, data in LDA['topics'].items():100 topic_words = set(data['words'][:20])101 score = 0102 matches = []103 for word, prob in zip(data['words'][:20], data['probs'][:20]):104 if word in freq:105 score += freq[word] * prob106 matches.append(word)107 if score > 0:108 scores[tid] = score109 matches_by_topic[tid] = matches110 111 if not scores:112 return None, tokens, "No matching topics"113 114 # Normalize115 total = sum(scores.values())116 117 results = []118 for tid, score in sorted(scores.items(), key=lambda x: x[1], reverse=True)[:n]:119 results.append({120 'id': tid,121 'name': TOPIC_NAMES['lda'].get(tid, "Topic " + tid),122 'prob': (score / total) * 100,123 'words': LDA['topics'][tid]['words'][:6],124 'matches': matches_by_topic.get(tid, [])[:5]125 })126 127 return results, tokens, None128 129def analyze_lsa(text, n=5):130 tokens = preprocess(text)131 if len(tokens) < 3:132 return None, tokens, "Text too short"133 134 freq = Counter(tokens)135 136 scores = {}137 matches_by_topic = {}138 139 for tid, data in LSA['topics'].items():140 score = 0141 matches = []142 for word, weight in zip(data['words'][:20], data['weights'][:20]):143 if word in freq:144 score += freq[word] * abs(weight)145 matches.append(word)146 if score > 0:147 scores[tid] = score148 matches_by_topic[tid] = matches149 150 if not scores:151 return None, tokens, "No matching topics"152 153 total = sum(scores.values())154 155 results = []156 for tid, score in sorted(scores.items(), key=lambda x: x[1], reverse=True)[:n]:157 results.append({158 'id': tid,159 'name': TOPIC_NAMES['lsa'].get(tid, "Topic " + tid),160 'score': (score / total) * 100,161 'words': LSA['topics'][tid]['words'][:6],162 'matches': matches_by_topic.get(tid, [])[:5]163 })164 165 return results, tokens, None166 167def analyze_bert(text):168 tokens = preprocess(text)169 if len(tokens) < 3:170 return None, tokens, "Text too short"171 172 if not BERTOPIC.get('topics'):173 return None, tokens, "BERTopic not available"174 175 token_set = set(tokens)176 177 best_tid = None178 best_score = 0179 best_matches = []180 181 for tid, data in BERTOPIC['topics'].items():182 topic_words = set(data['words'][:15])183 matches = token_set.intersection(topic_words)184 if len(matches) > best_score:185 best_score = len(matches)186 best_tid = tid187 best_matches = list(matches)188 189 if best_tid:190 return {191 'id': best_tid,192 'name': BERTOPIC['topics'][best_tid]['name'],193 'words': BERTOPIC['topics'][best_tid]['words'][:8],194 'matches': best_matches[:6]195 }, tokens, None196 197 return None, tokens, "No matching topic"198 199def format_lda(results, tokens, error):200 if error:201 return '<div style="background:#fef3c7;padding:12px;border-radius:8px;color:#92400e;">' + error + '</div>'202 203 html = ""204 for r in results:205 prob_str = "{:.1f}".format(r['prob'])206 bar_width = str(min(r['prob'], 100))207 208 kw = ""209 for w in r['words']:210 kw += '<span style="background:#dbeafe;color:#1e40af;padding:2px 6px;border-radius:4px;margin:2px;font-size:0.85em;">' + w + '</span> '211 212 matches = ""213 if r['matches']:214 for w in r['matches']:215 matches += '<span style="background:#93c5fd;color:#1e40af;padding:2px 8px;border-radius:4px;margin:2px;">' + w + '</span> '216 else:217 matches = '<span style="color:#9ca3af;">none</span>'218 219 html += '<div style="background:white;border-left:4px solid #3b82f6;padding:12px;margin-bottom:10px;border-radius:8px;">'220 html += '<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px;">'221 html += '<strong style="color:#1e40af;">' + r['name'] + '</strong>'222 html += '<span style="background:#3b82f6;color:white;padding:4px 12px;border-radius:20px;font-weight:600;">' + prob_str + '%</span>'223 html += '</div>'224 html += '<div style="background:#e5e7eb;border-radius:8px;height:8px;margin-bottom:8px;"><div style="background:#3b82f6;height:100%;border-radius:8px;width:' + bar_width + '%;"></div></div>'225 html += '<div style="font-size:0.85em;margin-bottom:4px;"><b>Topic words:</b> ' + kw + '</div>'226 html += '<div style="font-size:0.9em;"><b>Your matches:</b> ' + matches + '</div>'227 html += '</div>'228 229 return html230 231def format_lsa(results, tokens, error):232 if error:233 return '<div style="background:#fef3c7;padding:12px;border-radius:8px;color:#92400e;">' + error + '</div>'234 235 html = ""236 for r in results:237 score_str = "{:.1f}".format(r['score'])238 bar_width = str(min(r['score'], 100))239 240 kw = ""241 for w in r['words']:242 kw += '<span style="background:#d1fae5;color:#065f46;padding:2px 6px;border-radius:4px;margin:2px;font-size:0.85em;">' + w + '</span> '243 244 matches = ""245 if r['matches']:246 for w in r['matches']:247 matches += '<span style="background:#6ee7b7;color:#065f46;padding:2px 8px;border-radius:4px;margin:2px;">' + w + '</span> '248 else:249 matches = '<span style="color:#9ca3af;">none</span>'250 251 html += '<div style="background:white;border-left:4px solid #10b981;padding:12px;margin-bottom:10px;border-radius:8px;">'252 html += '<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px;">'253 html += '<strong style="color:#065f46;">' + r['name'] + '</strong>'254 html += '<span style="background:#10b981;color:white;padding:4px 12px;border-radius:20px;font-weight:600;">' + score_str + '%</span>'255 html += '</div>'256 html += '<div style="background:#e5e7eb;border-radius:8px;height:8px;margin-bottom:8px;"><div style="background:#10b981;height:100%;border-radius:8px;width:' + bar_width + '%;"></div></div>'257 html += '<div style="font-size:0.85em;margin-bottom:4px;"><b>Topic words:</b> ' + kw + '</div>'258 html += '<div style="font-size:0.9em;"><b>Your matches:</b> ' + matches + '</div>'259 html += '</div>'260 261 return html262 263def format_bert(result, tokens, error):264 if error:265 return '<div style="background:#fef3c7;padding:12px;border-radius:8px;color:#92400e;">' + error + '</div>'266 267 if not result:268 return '<div style="background:#fef3c7;padding:12px;border-radius:8px;color:#92400e;">No match found</div>'269 270 kw = ""271 for w in result['words']:272 kw += '<span style="background:#ede9fe;color:#5b21b6;padding:2px 6px;border-radius:4px;margin:2px;font-size:0.85em;">' + w + '</span> '273 274 matches = ""275 if result['matches']:276 for w in result['matches']:277 matches += '<span style="background:#c4b5fd;color:#5b21b6;padding:2px 8px;border-radius:4px;margin:2px;">' + w + '</span> '278 else:279 matches = '<span style="color:#9ca3af;">none</span>'280 281 name = str(result['name'])[:50]282 283 html = '<div style="background:white;border-left:4px solid #8b5cf6;padding:12px;border-radius:8px;">'284 html += '<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:8px;">'285 html += '<strong style="color:#5b21b6;">' + name + '</strong>'286 html += '<span style="background:#8b5cf6;color:white;padding:4px 12px;border-radius:20px;font-weight:600;">Topic ' + str(result['id']) + '</span>'287 html += '</div>'288 html += '<div style="font-size:0.85em;margin-bottom:4px;"><b>Topic words:</b> ' + kw + '</div>'289 html += '<div style="font-size:0.9em;"><b>Your matches:</b> ' + matches + '</div>'290 html += '</div>'291 292 return html293 294def analyze(text_input, file_input, model_choice, num_topics):295 text = ((text_input or "") + " " + read_file(file_input)).strip()296 297 if len(text) < 50:298 msg = '<div style="background:#fef3c7;padding:16px;border-radius:8px;text-align:center;color:#92400e;">Please enter at least 50 characters</div>'299 return msg, "", "", ""300 301 tokens = preprocess(text)302 bigrams = len([t for t in tokens if '_' in t])303 in_vocab = len([t for t in tokens if t in WORD2ID])304 305 # Stats306 stats = '<div style="display:grid;grid-template-columns:repeat(4,1fr);gap:8px;margin-bottom:12px;">'307 stats += '<div style="background:white;padding:10px;border-radius:8px;text-align:center;"><div style="font-size:1.3em;font-weight:700;color:#667eea;">' + str(len(text)) + '</div><div style="font-size:0.8em;color:#6b7280;">Characters</div></div>'308 stats += '<div style="background:white;padding:10px;border-radius:8px;text-align:center;"><div style="font-size:1.3em;font-weight:700;color:#667eea;">' + str(len(tokens)) + '</div><div style="font-size:0.8em;color:#6b7280;">Tokens</div></div>'309 stats += '<div style="background:white;padding:10px;border-radius:8px;text-align:center;"><div style="font-size:1.3em;font-weight:700;color:#667eea;">' + str(bigrams) + '</div><div style="font-size:0.8em;color:#6b7280;">Bigrams</div></div>'310 stats += '<div style="background:white;padding:10px;border-radius:8px;text-align:center;"><div style="font-size:1.3em;font-weight:700;color:#667eea;">' + str(in_vocab) + '</div><div style="font-size:0.8em;color:#6b7280;">In Vocab</div></div>'311 stats += '</div>'312 313 # Show tokens314 sample = ""315 for t in tokens[:12]:316 sample += '<span style="background:#f3f4f6;padding:2px 6px;border-radius:4px;font-size:0.85em;margin:2px;">' + t + '</span> '317 if len(tokens) > 12:318 sample += '<span style="color:#6b7280;">+' + str(len(tokens)-12) + ' more</span>'319 stats += '<div style="background:white;padding:10px;border-radius:8px;margin-bottom:12px;"><b>Processed tokens:</b> ' + sample + '</div>'320 321 n = int(num_topics)322 lda_html = lsa_html = bert_html = ""323 324 if model_choice in ["All", "LDA"]:325 r, t, e = analyze_lda(text, n)326 lda_html = '<h3 style="color:#3b82f6;margin-bottom:8px;">LDA Results</h3>' + format_lda(r, t, e)327 328 if model_choice in ["All", "LSA"]:329 r, t, e = analyze_lsa(text, n)330 lsa_html = '<h3 style="color:#10b981;margin-bottom:8px;">LSA Results</h3>' + format_lsa(r, t, e)331 332 if model_choice in ["All", "BERTopic"]:333 r, t, e = analyze_bert(text)334 bert_html = '<h3 style="color:#8b5cf6;margin-bottom:8px;">BERTopic Match</h3>' + format_bert(r, t, e)335 336 return stats, lda_html, lsa_html, bert_html337 338# Interface339with gr.Blocks(title="Topic Analyzer", theme=gr.themes.Soft()) as demo:340 gr.Markdown("# Topic Analyzer")341 gr.Markdown("Using YOUR trained models: Dictionary, Bigrams, LDA, LSA, BERTopic")342 343 with gr.Row():344 with gr.Column():345 text_input = gr.Textbox(label="Enter Text", placeholder="Type or paste text here...", lines=8)346 file_input = gr.File(label="Or Upload File (TXT, PDF, DOCX)")347 model_choice = gr.Radio(["All", "LDA", "LSA", "BERTopic"], value="All", label="Model")348 num_topics = gr.Slider(3, 5, 5, step=1, label="Number of Topics")349 btn = gr.Button("Analyze", variant="primary", size="lg")350 351 with gr.Column():352 stats_out = gr.HTML()353 lda_out = gr.HTML()354 lsa_out = gr.HTML()355 bert_out = gr.HTML()356 357 gr.Markdown("### Examples")358 gr.Examples([359 ["President Biden announced new economic policies at the White House today focusing on jobs.", None, "All", 5],360 ["Manchester United defeated Liverpool 3-2 in an exciting Premier League match.", None, "All", 5],361 ["Apple unveiled its new iPhone with advanced AI features and improved camera.", None, "All", 5],362 ["Police arrested suspects in connection with a bank robbery in downtown Manhattan.", None, "All", 5],363 ], inputs=[text_input, file_input, model_choice, num_topics])364 365 btn.click(analyze, [text_input, file_input, model_choice, num_topics], [stats_out, lda_out, lsa_out, bert_out])366 367demo.launch(server_name="0.0.0.0", server_port=7860)368