qhnprof/NEWS_Signals
0
1import gradio as gr2from statsmodels.tsa.arima.model import ARIMA3from statsmodels.tsa.forecasting.stl import STLForecast4from statsmodels.tsa.seasonal import STL5import pandas as pd6import plotly.express as px7import json8import pickle9import numpy as np10import plotly.graph_objects as go11from plotly.subplots import make_subplots12 13 14# newsTags=["Politics Diplomacy Government",'Sanctions', "Religion Islam",'Women girl female', "Economy Finance Business",'Violance Demonstration Death Crime', "Sports Game", "Entertainment Art Fashion", "Health Healthcare Medicine", "Technology Science Education", "Environment Weather Agriculture Nature", "Military Defence War Attack", "Media Journalism Socialnetwork", "Movies Music Books Literature", "Advertising Marketing"]15# newsTags=[x.lower() for x in newsTags]16 17newsAgenciesDic={'bbc.json':'BBC Persian',18 'fars.json':'Fars News Agency',19 'irna.json':'IRNA News Agency',20 'iranint.json':'Iran International',21 'voa.json':'VOA News',22 'khabar.json':'Khabar Online',23 'tabnak.json':'Tabnak News Agency',24 'mamlekat.json':'Mamlekate Telegram Channel',25 }26 27tags=["Politics Diplomacy Government",'Sanctions', "Religion Islam",'Women girl female', 28"Economy Finance Business",'Violance Demonstration Death Crime', "Sports Game",29"Entertainment Art Fashion", "Health Healthcare Medicine", "Technology Science Education", 30"Environment Weather Agriculture Nature", "Military Defence War Attack", "Media Journalism Socialnetwork",31"Movies Music Books Literature", "Advertising Marketing"]32emotions = [ "Joy", "Love", "Gratitude", "Excitement", "Calmness", 33 "Nostalgia", "Sadness", "Anger", "Fear", "Disgust", "Frustration", 34 "Guilt", "Shame", "Surprise"]35sentiments=['positive sentiment','negative sentiment']36 37#s_tags_en=['social','economic','international','political','scientific','cultural','sport','medical']38s_tags=['اجتماعی','اقتصادی','بین الملل','سیاسی','علمی فناوری','فرهنگی هنری','ورزشی','پزشکی']39s_sentiments=['Negative','Neutral','Positive']40s_emotions=['furious', 'angry', 'neutral', 'happy', 'delighted']41 42def news_signal(news_id_1,news_id_2,topics,allTopics,allTopicsAvg,dfselector,period,rangeYear1,rangeYear2,chart_type,normalize,modelSel,modelType):43 #load df from disk44 dfselector=str(4-dfselector)45 #get key for news_id_1 from newsAgenciesDic46 title=news_id_147 title2=''48 news_id_1=list(newsAgenciesDic.keys())[list(newsAgenciesDic.values()).index(news_id_1)]49 if news_id_2!='None':50 title2=news_id_251 news_id_2=list(newsAgenciesDic.keys())[list(newsAgenciesDic.values()).index(news_id_2)]52 53 if allTopicsAvg=='True':54 # load df from disk pickle55 f1='dfs/avg_'+news_id_1+'.pkl'56 with open(f1, 'rb') as f:57 df1 = pickle.load(f)58 59 df1.index=pd.to_datetime(df1.index)60 61 if news_id_2!='None':62 f2='dfs/avg_'+news_id_2+'.pkl'63 with open(f2, 'rb') as f:64 df2 = pickle.load(f)65 df2.index=pd.to_datetime(df2.index)66 df3=pd.merge(df1,df2, left_index=True, right_index=True, how='inner',suffixes=('_x', '_y')).apply(lambda x:util.cos_sim(x['emb_x'],x['emb_y']).numpy().flatten()[0],axis=1)67 df3.columns=[title,title2]68 df3=df3.resample('M').mean()69 df3=df3[(df3.index.year>=2021) ]70 df3=df3*10071 fig=px.line(df3,title='Similarity between '+title+' and '+title2+' news agencies',labels={'value':'Similarity (%)','index':'Date'})72 return fig73 74 75 76 model_sel='u_'77 if modelSel=='Supervised':78 model_sel=''79 model_type='t_'80 if modelType=='Sentiment':81 model_type='s_'82 if modelType=='Emotion':83 model_type='e_'84 85 df=pd.read_csv('dfs/'+model_sel+'scores_'+model_type+news_id_1+'.csv',index_col=0)86 87 df.index=pd.to_datetime(df.index)88 #filter df index by year89 df = df.sort_index()90 91 df=df.loc[str(rangeYear1):str(rangeYear2)]92 93 if news_id_2!='None':94 df2=pd.read_csv('dfs/'+model_sel+'scores_'+model_type+news_id_2+'.csv',index_col=0)95 df2.index=pd.to_datetime(df2.index)96 df2 = df2.sort_index()97 df2=df2.loc[str(rangeYear1):str(rangeYear2)]98 99 #group by month100 if period=='Week':101 period='W'102 elif period=='Month':103 period='M'104 else:105 period='Y'106 107 # df_grouped=df.resample(period).sum()108 109 if normalize=='True':110 #convert each group into percentage111 #df_grouped=df_grouped.div(df_grouped.sum(axis=1), axis=0)112 df_grouped=df.resample(period).mean()113 if news_id_2!='None':114 df_grouped2=df2.resample(period).mean()115 else:116 df_grouped=df.resample(period).sum()117 if news_id_2!='None':118 df_grouped2=df2.resample(period).sum()119 120 121 122 # print(topics)123 # fig = px.line(df_m[[topics]]) 124 if len(topics)==0:125 return px.line()126 if allTopics=='False':127 df_grouped=df_grouped[topics]128 if news_id_2!='None':129 df_grouped2=df_grouped2[topics]130 #topics=[t.lower() for t in tags]131 132 if news_id_2!='None':133 # rename all columns by attaching news_id_2 to them134 df_grouped.columns=[str(col) + title for col in df_grouped.columns]135 df_grouped2.columns=[str(col) + title2 for col in df_grouped2.columns]136 df_join=df_grouped.join(df_grouped2,how='outer')137 138 #choose different colors for each line139 if chart_type=='Line':140 fig=px.line(df_grouped)141 if news_id_2!='None':142 fig=px.line(df_join)143 144 else:145 #bar chart146 fig=px.bar(df_grouped)147 if news_id_2!='None':148 fig=px.bar(df_join)149 150 #fig.add_trace(px.bar(df_grouped),row=1,col=1)151 152 fig.update_traces(mode='lines+markers')153 154 fig.update_layout(title='News Agency '+title+' ,'+title2, xaxis_title="Date", yaxis_title="News Signal",height=1000)155 return fig156 157 158 159def news_signal_predict(news_id_2,topics_2,dfselector_2,period_2,rangeYear1_2,rangeYear2_2,normalize_2,predMode):160 dfselector_2=str(4-dfselector_2)161 news_id_2=list(newsAgenciesDic.keys())[list(newsAgenciesDic.values()).index(news_id_2)]162 # df=pd.read_csv('df'+dfselector_2+'_'+news_id_2+'.csv',index_col=0)163 model_sel='u_'164 165 model_type='t_'166 df=pd.read_csv('dfs/'+model_sel+'scores_'+model_type+news_id_2+'.csv',index_col=0)167 168 df.index=pd.to_datetime(df.index)169 df = df.sort_index()170 171 df=df.loc[str(rangeYear1_2):str(rangeYear2_2)]172 173 pred_len=10174 if period_2=='Week':175 period_2='W'176 pred_len=pred_len*7177 elif period_2=='Month':178 period_2='M'179 180 181 182 df_grouped=df.resample(period_2).sum()183 184 if normalize_2=='True':185 df_grouped=df_grouped.div(df_grouped.sum(axis=1), axis=0)186 187 if predMode=='STL':188 stl = STL(df_grouped[topics_2], seasonal=13, period=7, robust=True)189 res = stl.fit()190 fig = make_subplots(rows=4, cols=1,shared_xaxes=True)191 fig.add_trace(go.Scatter(x=res.observed.index, y=res.observed.values, mode='lines', name='Original'),row=1, col=1)192 fig.add_trace(go.Scatter(x=res.trend.index, y=res.trend.values, mode='lines', name='Trend'),row=2, col=1)193 fig.add_trace(go.Scatter(x=res.seasonal.index, y=res.seasonal.values, mode='lines', name='Seasonal'),row=3, col=1)194 fig.add_trace(go.Scatter(x=res.resid.index, y=res.resid.values, mode='lines', name='Residual'),row=4, col=1)195 #hide plotly tools196 fig.update_layout(showlegend=False)197 #add title to each subplot198 fig.update_yaxes(row=1,title_text="Original")199 fig.update_yaxes(row=2,title_text="Trend")200 fig.update_yaxes(row=3,title_text="Seasonal")201 fig.update_yaxes(row=4,title_text="Residual")202 fig.update_layout(title='STL Forecast'+topics_2+' for '+news_id_2,height=800)203 return fig204 205 elif predMode=='ARIMA':206 stlf = STLForecast(df_grouped[topics_2], ARIMA, model_kwargs=dict(order=(1, 1, 0), trend="t") ,robust=True)207 res = stlf.fit()208 forcast=res.forecast(pred_len)209 #use plotly210 fig = px.line(x=df_grouped[topics_2].index,y=df_grouped[topics_2].values)211 fig.add_scatter(x=forcast.index,y=forcast.values)212 fig.update_layout(title='ARIMA Forecast'+topics_2+' for '+news_id_2)213 return fig214 elif predMode.find('CTG')!=-1:215 216 allDFs=[]217 for newsID in list(newsAgenciesDic.keys()):218 df=pd.read_csv('dfs/'+model_sel+'scores_'+model_type+newsID+'.csv',index_col=0)219 220 #df=pd.read_csv('df'+dfselector_2+'_'+newsID+'.csv',index_col=0)221 df.index=pd.to_datetime(df.index)222 df = df.sort_index()223 224 df=df.loc[str(rangeYear1_2):str(rangeYear2_2)]225 226 #if normalize_2=='True':227 # df_grouped=df_grouped.div(df_grouped.sum(axis=1), axis=0)228 if normalize_2=='True':229 df_grouped=df.resample('M').mean()230 else:231 df_grouped=df.resample('M').sum()232 233 #print(newsID,df_grouped[topics_2].head())234 235 allDFs.append(df_grouped[topics_2])236 #merge all dataframes on index237 dfs=pd.concat(allDFs,axis=1)238 #rename columns239 dfs.columns=newsAgenciesDic.values()240 #remove the last month data241 dfs=dfs.iloc[:-1]242 #remove rows with missing values243 dfs.fillna(0,inplace=True)244 #print('ALL',len(allDFs),dfs.head())245 246 if predMode=='CTG1':247 #create a subplot in each row248 fig=make_subplots(rows=len(dfs.columns),cols=1,shared_xaxes=True,subplot_titles=list(newsAgenciesDic.values()))249 250 for i in range(len(dfs.columns)):251 fig.add_trace(go.Scatter(x=dfs.index,y=dfs.iloc[:,i],mode='lines',name=list(newsAgenciesDic.values())[i]),row=i+1,col=1)252 #hide plotly tools253 fig.update_layout(showlegend=False,height=1200,title='NEWS Signal Comparison for Topic: '+topics_2)254 else:255 256 #return plotly figure257 fig=px.line(dfs)258 fig.update_layout(title='NEWS Signal Comparison for Topic: '+topics_2)259 260 261 262 return fig#dfs.plot()263 264 265 266 #plt.plot(df_grouped[topics_2])267 #plt.plot(forcast)268 #increase size of plot269 #plt.rcParams["figure.figsize"] = (20,10)270 271def fn_model_sel(modelSel,modelType):272 print(modelSel,modelType)273 if modelSel=='Self-Supervised':274 if modelType=='Tag':275 return topics.update(choices=tags,value=tags[0])276 if modelType=='Sentiment':277 return topics.update(choices=sentiments,value=sentiments[0])278 if modelType=='Emotion':279 return topics.update(choices=emotions,value=emotions[0])280 else:281 if modelType=='Tag':282 return topics.update(choices=s_tags,value=s_tags[0])283 if modelType=='Sentiment':284 return topics.update(choices=s_sentiments,value=s_sentiments[0])285 if modelType=='Emotion':286 return topics.update(choices=s_emotions,value=s_emotions[0])287 288 289 290def news_signal_correlation(news_id_3,news_id_4,rangeYear1_3,rangeYear2_3,modelType,modelSel):291 model_sel='u_'292 if modelSel=='Supervised':293 model_sel=''294 model_type='t_'295 if modelType=='Sentiment':296 model_type='s_'297 if modelType=='Emotion':298 model_type='e_'299 300 news_id_3=list(newsAgenciesDic.keys())[list(newsAgenciesDic.values()).index(news_id_3)]301 df1=pd.read_csv('dfs/'+model_sel+'scores_'+model_type+news_id_3+'.csv',index_col=0)302 df1.index=pd.to_datetime(df1.index)303 df = df.sort_index()304 305 df1=df1.loc[str(rangeYear1_3):str(rangeYear2_3)]306 307 if news_id_4=='none':308 fig = go.Figure()309 fig.add_trace(go.Heatmap(z=df1.corr(), x=df1.columns, y=df1.columns))310 fig.update_layout(title_text=modelSel+' '+modelType+' '+"Correlation Heatmap "+newsAgenciesDic[news_id_3],height=1000)311 return fig312 else:313 314 news_id_4=list(newsAgenciesDic.keys())[list(newsAgenciesDic.values()).index(news_id_4)]315 df2=pd.read_csv('dfs/'+model_sel+'scores_'+model_type+news_id_4+'.csv',index_col=0)316 df2.index=pd.to_datetime(df2.index)317 df = df.sort_index()318 319 df2=df2.loc[str(rangeYear1_3):str(rangeYear2_3)]320 321 corr=df1.corrwith(df2)322 #plotly bar plot corr323 fig=px.bar(corr)324 fig.update_layout(showlegend=False,title_text=modelSel+' '+modelType+' ' +"Correlation Chart between "+325 newsAgenciesDic[news_id_3]+' and '+ newsAgenciesDic[news_id_4],height=1000)326 return fig327 328with gr.Blocks(title='QNEWS') as demo:329 gr.Markdown('# NEWS SIGNAL ANALYSIS')330 331 with gr.Tab('Signal Analysis'):332 with gr.Row(variant='panel'):333 with gr.Column():334 news_id_1 = gr.Dropdown(list(newsAgenciesDic.values()),label='Select News Agency',value=list(newsAgenciesDic.values())[0])335 new_list=list(newsAgenciesDic.values())336 new_list.append('None')337 news_id_2 = gr.Dropdown(new_list,label='Select Second News Agency',value='None')338 modelType = gr.Radio(['Tag','Sentiment','Emotion'],label='Model Type',value='Tag')339 modelSel = gr.Radio(['Self-Supervised','Supervised'],label='Model Selection',value='Self-Supervised')340 topics = gr.CheckboxGroup(tags,label='Select Topic\'s Signal',value=tags[0],multiselect=True)341 allTopics = gr.Radio(['True','False'],label='Select All Topics',value='False')342 allTopicsAvg=gr.Radio(['True','False'],label='Select All Topics Average',value='False')343 344 345 with gr.Column(scale=0.5): 346 period = gr.Radio(['Week','Month','Year'],label='Select Grouping Period',value='Month')347 chart_type = gr.Radio(['Line','Bar'],label='Select Chart Type',value='Line')348 normalize = gr.Radio(['True','False'],label='Signal Normalization',value='False')349 dfselector = gr.Slider(minimum=1,maximum=3,step=1,value=2,label='Select Signal Sensitivity from 1 to 3',visible=False)350 351 rangeYear1 = gr.Slider(minimum=2015,maximum=2022,step=1,value=2015,label='From Year')352 rangeYear2 = gr.Slider(minimum=2016,maximum=2023,step=1,value=2023,label='To Year')353 # btn_predict = gr.Radio(['a','b','c'],label='Signal Mode',value='a')354 bt1=gr.Button()355 356 357 inputs=[news_id_1,news_id_2,topics,allTopics,allTopicsAvg,dfselector,period,rangeYear1,rangeYear2,chart_type,normalize,modelSel,modelType]358 359 with gr.Row():360 output = gr.Plot()361 362 modelType.change(fn_model_sel,[modelSel,modelType],topics)363 modelSel.change(fn_model_sel,[modelSel,modelType],topics)364 normalize.change(news_signal,inputs, output)365 chart_type.change(news_signal,inputs, output)366 rangeYear2.change(news_signal,inputs, output)367 rangeYear1.change(news_signal,inputs, output)368 period.change(news_signal,inputs, output)369 dfselector.change(news_signal,inputs, output)370 allTopics.change(news_signal,inputs, output)371 topics.change(news_signal,inputs, output)372 news_id_1.change(news_signal,inputs, output)373 bt1.click(news_signal,inputs, output)374 375 # btn_predict.change(fn_predict,[btn_predict],outputs= [row1,row2])376 demo.load(news_signal,inputs, output)377 with gr.Tab('Signal Forcasting'):378 with gr.Row(variant='panel'):379 with gr.Column():380 news_id_2 = gr.Dropdown(list(newsAgenciesDic.values()),label='Select News Agency',value=list(newsAgenciesDic.values())[0])381 topics_2 = gr.Dropdown(tags,label='Select Topic\'s Signal',value=tags[0],multiselect=False)382 with gr.Row():383 with gr.Column():384 rangeYear1_2 = gr.Slider(minimum=2015,maximum=2022,step=1,value=2015,label='From Year')385 with gr.Column():386 rangeYear2_2 = gr.Slider(minimum=2016,maximum=2023,step=1,value=2023,label='To Year')387 with gr.Column(): 388 period_2 = gr.Radio(['Week','Month'],label='Select Grouping Period',value='Month')389 normalize_2 = gr.Radio(['True','False'],label='Signal Normalization',value='False')390 dfselector_2 = gr.Slider(minimum=1,maximum=3,step=1,value=2,label='Select Signal Sensitivity from 1 to 3')391 predMode = gr.Radio(['STL','ARIMA','CTG1','CTG2'],label='Select Model Type',value='STL')392 393 submit_btn = gr.Button("Signal Analysis")394 395 inputs=[news_id_2,topics_2,dfselector_2,period_2,rangeYear1_2,rangeYear2_2,normalize_2,predMode]396 with gr.Row():397 398 output2 = gr.Plot()399 400 submit_btn.click(news_signal_predict,inputs, output2)401 402 with gr.Tab('Signal Correlation'):403 with gr.Row(variant='panel'):404 405 newsList=list(newsAgenciesDic.values())406 news_id_3 = gr.Dropdown(newsList,label='Select News Agency 1',value=newsList[0])407 newsList.insert(0,'none')408 news_id_4 = gr.Dropdown(newsList,label='Select News Agency 2',value=newsList[0])409 modelType = gr.Radio(['Tag','Sentiment','Emotion'],label='Model Type',value='Tag')410 modelSel = gr.Radio(['Self-Supervised','Supervised'],label='Model Selection',value='Self-Supervised')411 with gr.Row():412 with gr.Column():413 rangeYear1_3 = gr.Slider(minimum=2015,maximum=2022,step=1,value=2015,label='From Year')414 with gr.Column():415 rangeYear2_3 = gr.Slider(minimum=2016,maximum=2023,step=1,value=2023,label='To Year')416 bt3=gr.Button()417 with gr.Row():418 output3 = gr.Plot()419 bt3.click(news_signal_correlation,inputs=[news_id_3,news_id_4,rangeYear1_3,rangeYear2_3,modelType,modelSel],outputs=output3)420 421 422 423 424 425server=demo.launch()426