o6Dool/Search_Using_Topic_Modeling
0
1import streamlit as st
2import pandas as pd
3import matplotlib.pyplot as plt
4
5import gensim
6from gensim.corpora import Dictionary
7from gensim.models.ldamulticore import LdaMulticore
8import pathlib
9
10import nltk
11from nltk.tokenize import word_tokenize
12from nltk.stem import WordNetLemmatizer
13from nltk.corpus import stopwords
14
15import numpy as np
16from gensim.models import LdaModel
17import pickle
18from wordcloud import WordCloud
19
20
21
22class blind_search_engine:
23
24 def __init__(self):
25
26 nltk.download('punkt')
27 nltk.download('wordnet')
28 nltk.download('stopwords')
29 self.lda_model, self.df, self.dictionary, self.topicid_to_ids = self.load()
30 self.lemmatizer = WordNetLemmatizer()
31 self.stop_words = set(stopwords.words("english"))
32
33
34
35
36
37
38
39
40 def preprocess(self, text):
41 text = text.lower()
42 words = word_tokenize(text)
43 words = [self.lemmatizer.lemmatize(word) for word in words if word not in self.stop_words]
44 return words
45
46 def load(self):
47 chunks = []
48
49 df_address = str(pathlib.Path(__file__).parent.resolve()) + '/saved_data/sorted_df.csv'
50
51
52 for chunk in pd.read_csv(df_address, chunksize=10000):
53 chunks.append(chunk)
54
55 loaded_df = pd.concat(chunks, ignore_index=True)
56
57
58 dict_address = str(pathlib.Path(__file__).parent.resolve()) + '/saved_data/dictionary.dict'
59 loaded_dictionary = Dictionary.load(dict_address)
60
61
62 model_address = str(pathlib.Path(__file__).parent.resolve()) + '/saved_data/lda_model_80.model'
63 loaded_lda_model = LdaModel.load(model_address)
64
65 #topicid_toids_address = str(pathlib.Path(__file__).parent.resolve()) + '/saved_data/topicid_to_ids.pkl'
66 topicid_toids_address = str(pathlib.Path(__file__).parent.resolve()) + '/saved_data/new_topicid_to_ids_80.pkl'
67 with open(topicid_toids_address, 'rb') as f:
68 loaded_topicid_to_ids = pickle.load(f)
69
70 return loaded_lda_model, loaded_df, loaded_dictionary, loaded_topicid_to_ids
71
72 def bow_to_topicid(self, bow, no_lower_than=0.2, num=3):
73 p_topics = self.lda_model.get_document_topics(bow, minimum_probability=no_lower_than)
74 top = sorted(p_topics, key=lambda x: x[1], reverse=True)[:num]
75 return top
76
77 def search_to_topicid(self, search, no_lower_than=0.2, num=3):
78 preprocess_search = self.preprocess(search)
79 bow_search = self.dictionary.doc2bow(preprocess_search)
80 top = self.bow_to_topicid(bow_search, no_lower_than, num)
81 return top
82
83 def id_to_contents(self, id):
84 result = self.df.loc[self.df['id'] == id]
85 return result
86
87 def search_results(self, top, num_result=10, topics=3):
88
89 results = pd.DataFrame()
90
91
92 for topicid, prob_ in top[:topics]:
93 ids = self.topicid_to_ids[topicid]
94 probs = []
95 for id,prob in ids:
96 next = self.id_to_contents(id)
97 probs.append(prob)
98 if num_result == 0:
99 break
100 elif next is None:
101 break
102 else:
103 num_result = num_result - 1
104
105 results = pd.concat([results, next], axis=0)
106
107 return results,probs
108
109
110
111 def search(self, text: str, no_lower_than=0.2, num=3, num_result=10, topics=3):
112 top = self.search_to_topicid(text, no_lower_than, num)
113 results,probs = self.search_results(top, num_result, topics)
114 return results,probs
115
116 def turnOn(self,no_lower_than=0.2,num_result=10,topics=3):
117 x = input('Search >>: ')
118 while x != 'off' and x != 'end' and x!= 'close':
119
120 results = self.search(x,no_lower_than,num=3,num_result=10,topics=3)
121 print(results)
122 x = input('Search >>: ')
123 print('Search Ended')
124
125
126def create_topic_trend_table(df,n_topic,save = False):
127 topic_trend = 0
128 #cái try này ko quan trọng lắm, vì t để máy của bọn m và máy t khác nhau nên để thế thôi
129 try:
130 topic_trend = df.groupby(['main_topic', 'int_dates']).size().to_frame().unstack()
131 except:
132 topic_trend = df.groupby(['main_topic_80', 'int_dates']).size().to_frame().unstack()
133 topic_trend = topic_trend.fillna(0)
134 #soTopic = len(df['main_topic'].unique())
135 tempt = topic_trend.loc[:,0]
136 if save:
137 tempt.to_csv(f"topic_trend_day_{n_topic}.csv",index=False)
138 return tempt
139
140
141def trend_by_day(start_date=None,end_date=None,link=None):
142 # tempt = 0
143 # try:
144 if link == None:
145 link = str(pathlib.Path(__file__).parent.resolve()) + "/saved_data/topic_trend_day_80.csv"
146 tempt = pd.read_csv(link)
147 # except:
148 # raise Exception(f"can't seem to find this link :{link}\n ->{tempt}")
149 # kiến cho date luôn có nghĩa - luôn đủ dạng 8 chữ số
150 if start_date != None:
151 number = start_date
152 count = 0
153 while number > 1:
154 number //= 10
155 count += 1
156 if count == 4:
157 start_date = start_date*10000
158 elif count == 6:
159 start_date = start_date*100
160 # kiến cho date luôn có nghĩa - luôn đủ dạng 8 chữ số
161 if end_date != None:
162 number = end_date
163 count = 0
164 while number > 1:
165 number //= 10
166
167 count += 1
168 if count == 4:
169 end_date = end_date*10000 + 9999
170 elif count == 6:
171 end_date = end_date*100 + 99
172
173 # kiến cho date luôn có nghĩa - luôn đủ dạng 8 chữ số, ở đây là khoảng thời gian có trong dữ liệu
174 if start_date == None:
175 start_date = 20070523
176 if end_date == None:
177 end_date = 20240608
178 #if start_date > end_date:
179 # raise Exception('End date cannot be sooner than Start date')
180
181 # ko cần quan tâm cái này, thầy hỏi t sẽ trả lời
182 start_date = str(start_date)
183 end_date = str(end_date)
184 # trả về dữ liệu nằm trong khoảng thời gian được cho (từng ngày)
185 return tempt.loc[:,start_date:end_date]
186
187# tính tổng số bài của mỗi topic trong khoảng thời gian được cho
188def total(start_date=None,end_date=None,link=None):
189 maxtrix = trend_by_day(start_date,end_date,link)
190 return maxtrix.sum(axis = 1)
191
192def bar_visualize(vec, n_show,figsize = (20, 10)):
193 # fig, ax = plt.subplots(figsize=(10, 5))
194
195 # # Plot stacked bar chart
196 # vec.iloc[:n_show].plot(kind='bar', stacked=True, ax=ax)
197
198 # ax.set_xlabel("Date")
199 # ax.set_ylabel("Number of Articles")
200 # ax.legend(title='Categories')
201
202 # return fig, ax
203
204 fig, ax = plt.subplots(figsize=figsize)
205
206 # Transpose the DataFrame and plot stacked bar chart
207 vec.T.iloc[:, :n_show].plot(kind='bar', stacked=True, ax=ax)
208
209 ax.set_xlabel("Categories")
210 ax.set_ylabel("Number of Articles")
211 ax.set_title(f"Trend of")
212 ax.legend(title='Topics')
213
214 return fig, ax
215
216def lines_visualize(trend_data,n_show= 5, figsize = (20, 10)):
217
218 fig, ax = plt.subplots(figsize=figsize)
219 for i in range(len(trend_data[:n_show])):
220 x = trend_data.iloc[i].index
221 y = trend_data.iloc[i].values
222 ax.plot(x, y, label=f"Topic {i}")
223 ax.legend(bbox_to_anchor=(0.75, 1.15), ncol=10)
224 ax.set_xticklabels(trend_data.columns, rotation=90, ha='right')
225 return(fig,ax)
226
227def show_topic_total(total_topics,n_show = 5, figsize = (20,10)):
228
229 fig, ax = plt.subplots(figsize=figsize)
230 ax.bar(total_topics.index[:n_show], total_topics.values[:n_show])
231 ax.set_xlabel('Topic ID')
232 ax.set_ylabel('Number of Articles')
233 ax.set_xticks(total_topics.index[:n_show])
234
235 return fig, ax
236
237
238def trend(start_date, end_date,groupby='day',link = None):
239 # kiến cho date luôn có nghĩa - luôn đủ dạng 8 chữ số, ở đây là khoảng thời gian nằm trong dữ liệu
240 if start_date == None:
241 start_date = 20070523
242 if end_date == None:
243 end_date = 20240608
244
245 # groupby --> muốn gộp lại để show dữ liệu ra dưới dạng từng ngày một
246 # hay theo từng tháng, từng năm (được viết ở phía dưới cùng)
247
248 if groupby == 'day':
249 return trend_by_day(start_date,end_date,link)
250
251 # kiến cho date luôn có nghĩa - luôn đủ dạng 8 chữ số
252 end_count=0
253 start_count = 0
254 if start_date != None:
255 number = start_date
256 while number > 1:
257 number //= 10
258 start_count += 1
259 if start_count == 4:
260 start_date = start_date*10000 + 101
261 elif start_count == 6:
262 start_date = start_date*100 + 1
263
264 # kiến cho date luôn có nghĩa - luôn đủ dạng 8 chữ số
265 if end_date != None:
266 number = end_date
267 while number > 1:
268 number //= 10
269 end_count += 1
270 if end_count == 4:
271 end_date = end_date*10000 + 9999
272 elif end_count == 6:
273 end_date = end_date*100 + 99
274
275 # (trong trường hợp toàn bộ khoảng thời gian cung cấp không có dữ liệu nào)
276 # cảnh báo nếu thời gian vượt ngoài khoảng có trong dữ liệu, dừng chương trình luôn
277
278 if start_date > 20240608:
279 raise Exception('out of bound! There is no data above there!!')
280 if end_date < 20070523:
281 raise Exception('out of bound! There is nothing down here!')
282
283 #(trong trường hợp vẫn có một phần dữ liệu nằm trong khoảng thời gian cung cấp)
284 # giúp mô hình vẫn hoạt động được bình thường bằng cách đưa khoảng thời gian đó về hoàn toàn nằm trong khoảng thời gian có dữ liệu
285 if start_date < 20070523:
286 start_date = 20070523
287 if end_date > 20240608:
288 end_date = 20240608
289
290 #trong trường hợp ngày kết thúc nằm trước ngày bắt đầu, đảo lại vị trí của chúng (vì có thể nhỡ người dùng nhầm -> vẫn hoạt động bth)
291 if start_date > end_date:
292 tmp = start_date
293 start_date = end_date
294 end_date = tmp
295
296 if groupby != 'year' and groupby != 'month' and groupby != 'day':
297 raise Exception('grouping can only be by day,month or year')
298
299 # tách nhỏ tháng, và năm ra thành từng số một để dễ xử lý
300 yearstart = start_date//10000
301 yearend = end_date//10000
302 monthstart = int(max((start_date/100)%100,1))
303 monthend = int(min((end_date/100)%100,12))
304
305
306 tempts = pd.DataFrame()
307
308 # toàn bộ phần ở dưới đây chỉ là để đảm bảo rằng, dữ liệu trả về sẽ luôn nằm trong khoảng thời gian người dùng đưa ra
309 # và đúng theo kiểu gộp dữ liệu để hiện thị mà người dùng chọn
310 if groupby == 'year':
311 # if within same year
312 if yearstart == yearend:
313 tempt = total(start_date,end_date,link)
314 tempt = pd.DataFrame(tempt,columns=[f'{yearstart}'])
315 return tempt
316
317
318 #(start_date ---> end of the starting year)
319 tempt = total(start_date,yearstart,link)
320 tempt = pd.DataFrame(tempt,columns=[f'{yearstart}'])
321 tempts = pd.concat([tempts,tempt], axis = 1)
322
323 #(all the years in between start_date and end_date)
324 for i in range(yearstart+1,yearend):
325 tempt = total(i,i,link)
326 tempt = pd.DataFrame(tempt,columns=[f'{str(i)}'])
327 tempts = pd.concat([tempts,tempt], axis = 1)
328
329 #(start of end year --> end_date)
330 tempt = total(yearend,end_date,link)
331 tempt = pd.DataFrame(tempt,columns=[f'{yearend}'])
332 tempts = pd.concat([tempts,tempt], axis = 1)
333 return tempts
334
335 if groupby == 'month':
336
337
338 #if within the same year
339 if yearstart == yearend:
340 for month in range(monthstart,monthend+1):
341 _month = yearstart*10000+month*101
342 month_ = yearend*10000+month*100+32
343 tempt = total(max(_month,start_date),min(month_,end_date),link)
344
345 tempt = pd.DataFrame(tempt,columns=[f'{str(_month//100)}'])
346 tempts = pd.concat([tempts,tempt], axis = 1)
347 return tempts
348
349 #(start_date --> 12)
350 for month in range(monthstart,12+1):
351 _month = yearstart*10000 + month*100
352 month_ = yearstart*100 + month
353 tempt = total(max(start_date,_month),month_,link)
354
355
356 tempt = pd.DataFrame(tempt,columns=[f'{str(month_)}'])
357 tempts = pd.concat([tempts,tempt], axis = 1)
358
359 # all the years in between start_date and end_date
360 for year in range(yearstart+1,yearend):
361 for month in range(1,12 + 1):
362 tempt = total(year*100+month,year*100+month,link)
363
364 tempt = pd.DataFrame(tempt,columns=[f'{str(year*100+month)}'])
365 tempts = pd.concat([tempts,tempt], axis = 1)
366
367 #(1 - > end_date)
368 for month in range(1,monthend+1):
369 _month = yearend*100 + month
370 month_ = yearend*10000 + month*100 + 32
371 tempt = total(_month,min(end_date,month_),link)
372
373 tempt = pd.DataFrame(tempt,columns=[f'{str(_month)}'])
374 tempts = pd.concat([tempts,tempt], axis = 1)
375
376 return tempts
377
378
379def strip_spaces(text):
380
381 #Loại bỏ khoảng trắng ở đầu và cuối chuỗi.
382
383 return text.strip()
384
385def convert_dash(text):
386 return text.replace('\\','/')
387
388def read_text_from_file(file_path):
389 try:
390 with open(file_path, 'r') as file:
391 text = file.read()
392 return text
393 except FileNotFoundError:
394 return file_path
395 except Exception as e:
396 return f"An error occurred: {e}"
397
398def read_doc(text):
399 link = convert_dash(text)
400 text = read_text_from_file(link)
401 return text
402
403
404
405def doc_info(content,lda_address):
406
407 lda_model = LdaModel.load(lda_address)
408 dictionary = lda_model.id2word
409
410 lemmatizer = WordNetLemmatizer()
411 stop_words = set(stopwords.words("english"))
412 nltk.download('punkt')
413 nltk.download('wordnet')
414 nltk.download('stopwords')
415
416 content = content.lower()
417 content = word_tokenize(content)
418 content = [lemmatizer.lemmatize(word) for word in content if word not in stop_words]
419 bow = dictionary.doc2bow(content)
420
421
422 doc_topics = lda_model.get_document_topics(bow)
423 #topics = lda_model.show_topic()
424 return doc_topics
425
426import numpy as np
427import matplotlib.pyplot as plt
428from matplotlib.patches import Patch
429
430
431
432
433
434def show_topic_radar(total,n_show = 5,figsize = (20, 10)):
435
436
437 total_articles = total.sum()
438 topics = [(topic_id, count / total_articles) for topic_id, count in total.items()]
439
440 values = [probability for topic_id, probability in topics[:n_show]]
441 categories = [f'Topic {topic_id}' for topic_id, probability in topics[:n_show]]
442
443 num_categories = len(categories)
444 angles = np.linspace(0, 2 * np.pi, num_categories, endpoint=False)
445 values = np.concatenate((values, [values[0]]))
446 angles = np.concatenate((angles, [angles[0]]))
447
448 fig,ax = plt.subplots(figsize=figsize)
449
450 ax = fig.add_subplot(111, polar=True)
451
452
453
454 ax.plot(angles, values, 'o-', linewidth=2)
455 ax.fill(angles, values, alpha=0.25)
456 ax.set_xticks(angles[:-1], categories)
457
458 return fig, ax
459
460def doc_topic_radar(total,n_show = 5,figsize = (10, 10)):
461
462 total = total.values
463
464 values = [probability for topic_id, probability in total[:n_show]]
465 categories = [f'Topic {int(topic_id)}' for topic_id, probability in total[:n_show]]
466
467 num_categories = len(categories)
468 angles = np.linspace(0, 2 * np.pi, num_categories, endpoint=False)
469 values = np.concatenate((values, [values[0]]))
470 angles = np.concatenate((angles, [angles[0]]))
471
472 fig,ax = plt.subplots(figsize=figsize)
473
474 ax = fig.add_subplot(111, polar=True)
475
476 ax.plot(angles, values, 'o-', linewidth=2)
477 ax.fill(angles, values, alpha=0.25)
478 ax.set_xticks(angles[:-1], categories)
479
480
481 return fig, ax
482
483
484def generate_word_cloud(topic_id,link):
485 lda_model = LdaModel.load(link)
486
487
488 topic_words = lda_model.show_topic(topic_id, topn=20)
489 topic_dict = {word: prob for word, prob in topic_words}
490
491
492 wordcloud = WordCloud(width=800, height=400, background_color='white').generate_from_frequencies(topic_dict)
493
494
495 fig, ax = plt.subplots(figsize=(10, 5))
496 ax.imshow(wordcloud, interpolation='bilinear')
497 plt.axis('off')
498 ax.set_title(f'Topic {topic_id}')
499 return fig, ax
500
501
502import streamlit as st
503import pandas as pd
504import matplotlib.pyplot as plt
505
506
507
508
509
510
511#def main():
512st.title("Trend Analysis and Article Search")
513
514if 'search_engine' not in st.session_state:
515 placeholder = st.empty()
516 placeholder.write('Loading in data.........')
517 st.session_state.search_engine = blind_search_engine()
518 placeholder.write('Finished Loading in data')
519
520# Access the search engine from the session state
521search_engine = st.session_state.search_engine
522
523#keyword = st.text_input("Enter a keyword:")
524
525st.sidebar.header("Input Parameters")
526analysis_type = st.sidebar.radio("Select what you want to do", ("Trend Analysis", "Article Search","Document Analysis"))
527
528if analysis_type == "Trend Analysis":
529 start_date = st.sidebar.text_input("Start Date (YYYYMMDD)", value="20070523")
530 end_date = st.sidebar.text_input("End Date (YYYYMMDD)", value="20240608")
531 n_show = st.sidebar.text_input("Number of Top Topic to show", value="5")
532 groupby = st.sidebar.selectbox("Group By", ["day", "month", "year"])
533 link = st.sidebar.text_input("CSV Link (optional)",value = '')
534
535 if strip_spaces(link) == '':
536 link = None
537
538 try:
539 if n_show == '':
540 n_show = 5
541 n_show = int(n_show)
542 except:
543 raise Exception('value of number of topic showing is not appropriate')
544
545 if st.sidebar.button("Show Trend"):
546 try:
547 start_date = int(start_date)
548 end_date = int(end_date)
549 trend_data = trend(start_date, end_date, groupby, link)
550 st.write(trend_data)
551
552 st.subheader("Bar Visualization")
553 fig,ax = bar_visualize(trend_data,n_show)
554 st.pyplot(fig)
555
556 st.subheader("Line Visualization")
557 fig,ax = lines_visualize(trend_data,n_show)
558 st.pyplot(fig)
559
560 print(total(start_date,end_date,link))
561 top = total(start_date,end_date,link).sort_values(ascending=False)
562
563 st.subheader(f"Total topics from {start_date} to {end_date}")
564 fig,ax = show_topic_total(top,n_show)
565 st.pyplot(fig)
566
567 st.subheader(f"Rader Visualize of topics from {start_date} to {end_date}")
568 fig,ax = show_topic_radar(top,n_show)
569 st.pyplot(fig)
570
571
572
573 except Exception as e:
574 st.error(f"Error: {e}")
575
576elif analysis_type == "Article Search":
577 keyword = st.sidebar.text_input("Enter a keyword:")
578
579 if st.sidebar.button("Search"):
580 keyword = strip_spaces(keyword)
581 if keyword:
582 try:
583 results,probs = search_engine.search(keyword)
584 except:
585 raise Exception("Found Nothing")
586
587 if not results.empty:
588 i = 0
589 st.subheader(f"Results for '{keyword}':")
590 for index, row in results.iterrows():
591 st.markdown(f"Correlation Confidence: {round(probs[i]*100,1)}%")
592 i = i+ 1
593 st.markdown(f"ID: {row['id']}")
594 st.markdown(f"Title: {row['title']}")
595 st.markdown(f"DOI: {row['doi']}")
596 st.markdown(f"Abstract: {row['abstract'][:200]}...") # Displaying first 200 characters
597 with st.expander("Full Abstract"):
598 st.write(row['abstract'])
599 st.markdown("---")
600 else:
601 st.write("No results found.")
602 else:
603 st.write("Please enter a keyword.")
604
605elif analysis_type == "Document Analysis":
606 doc = st.sidebar.text_input("Enter a Document or Document's Path:", value="C:/Users\japan\OneDrive\Desktop\super_start.txt")
607 n_show = st.sidebar.text_input("Number of top relevant topics showing:", value = 5)
608 try:
609 n_show = int(n_show)
610 except TypeError:
611 raise Exception('Number of top relevant topics')
612
613
614 link = st.sidebar.text_input("LdaModel Address (Optional):")
615 link = strip_spaces(link)
616 if link == '':
617 link = str(pathlib.Path(__file__).parent.resolve()) + '/saved_data/lda_model_80.model'
618 link = convert_dash(link)
619
620 if st.sidebar.button("Analyze"):
621
622 if doc:
623 content = read_doc(doc)
624 doc_topics = doc_info(content,link)
625 topic = sorted(doc_topics, key=lambda x: x[1], reverse=True)[:n_show]
626 doc_topics = pd.DataFrame(topic)
627
628 topic_ids = [topic[index][0] for index in range(len(topic))]
629 top_probs = [f"{round(topic[index][1]*100,2)}%" for index in range(len(topic))]
630
631 topic_ids = pd.DataFrame(topic_ids, columns=['Topic ID'])
632 top_probs = pd.DataFrame(top_probs, columns=['percent'])
633 topics = pd.concat([topic_ids,top_probs],axis = 1)
634 st.write(topics)
635
636 st.subheader("Radar Chart for the Document' topics")
637 fig, ax = doc_topic_radar(doc_topics,n_show = len(doc_topics))
638 st.pyplot(fig)
639
640 st.subheader(f"Word Cloud of Topic ID: {topic[0][0]}")
641 fig, ax = generate_word_cloud(topic[0][0],link)
642 st.pyplot(fig)
643 else:
644 st.write("Please enter Document or Document's Path")
645
646
647
648