wldmr/transcriptifier-st-hf7
0
1from youtube_transcript_api import YouTubeTranscriptApi as yta2from youtube_transcript_api import NoTranscriptFound, TranscriptsDisabled3import streamlit as st4from yt_stats import YTstats5from datetime import datetime6import isodate7import pandas as pd8import deeppunkt9import time10import lexrank 11import mysheet 12 13def time_it(func):14 def wrapper(*args, **kwargs):15 start = time.time()16 result = func(*args, **kwargs)17 end = time.time()18 elapsed = end - start19 #st.write(f"Elapsed time: {end - start}")20 st.write('Load time: '+str(round(elapsed,1))+' sec')21 return result22 return wrapper23 24def reset_session():25 if 'punkt' in st.session_state:26 del st.session_state.punkt27 if 'extract' in st.session_state:28 del st.session_state.extract29 if 'channel_id' in st.session_state:30 del st.session_state.channel_id31 32def update_param_example():33 #st.session_state.url_vid = st.session_state.ex_vid34 video_id = get_id_from_link(st.session_state.ex_vid) 35 st.experimental_set_query_params(vid=video_id)36 reset_session()37 38def update_param_textinput():39 #st.session_state.url_vid = st.session_state.ti_vid40 video_id = get_id_from_link(st.session_state.ti_vid) 41 st.experimental_set_query_params(vid=video_id)42 reset_session()43 44def get_link_from_id(video_id):45 if "v=" not in video_id:46 return 'https://www.youtube.com/watch?v='+video_id47 else:48 return video_id49 50 51def get_id_from_link(link):52 if "v=" in link:53 return link.split("v=")[1].split("&")[0]54 elif len(link)==11:55 return link56 else:57 return "Error: Invalid Link."58 59# @st.cache(allow_output_mutation=True, suppress_st_warning=True)60# def retry_access_yt_object(url, max_retries=5, interval_secs=5, on_progress_callback=None):61# """62# Retries creating a YouTube object with the given URL and accessing its title several times63# with a given interval in seconds, until it succeeds or the maximum number of attempts is reached.64# If the object still cannot be created or the title cannot be accessed after the maximum number65# of attempts, the last exception is raised.66# """67# last_exception = None68# for i in range(max_retries):69# try:70# yt = YouTube(url, on_progress_callback=on_progress_callback)71# #title = yt.title # Access the title of the YouTube object.72# #views = yt.views73# return yt # Return the YouTube object if successful.74# except Exception as err:75# last_exception = err # Keep track of the last exception raised.76# st.write(f"Failed to create YouTube object or access title. Retrying... ({i+1}/{max_retries})")77# time.sleep(interval_secs) # Wait for the specified interval before retrying.78 79# # If the YouTube object still cannot be created or the title cannot be accessed after the maximum number of attempts, raise the last exception.80# raise last_exception81 82@st.cache_data()83def get_video_data(_yt, video_id):84 85 yt_img = f'http://img.youtube.com/vi/{video_id}/mqdefault.jpg'86 yt_img_html = '<img src='+yt_img+' width="250" height="150" />'87 yt_img_html_link = '<a href='+url+'>'+yt_img_html+'</a>'88 89 snippet = yt._get_single_video_data(video_id,'snippet')90 yt_publish_date = snippet['publishedAt']91 yt_title = snippet['title']92 yt_author = snippet['channelTitle']93 yt_channel_id = snippet['channelId'] 94 95 try:96 yt_keywords = snippet['tags']97 except:98 yt_keywords = []99 100 101 statistics = yt._get_single_video_data(video_id,'statistics')102 yt_views = statistics['viewCount']103 contentDetails = yt._get_single_video_data(video_id,'contentDetails')104 yt_length = contentDetails['duration']105 yt_length_isodate = isodate.parse_duration(yt_length)106 yt_length_isoformat = isodate.duration_isoformat(yt_length_isodate, "%H:%M:%S")[1:]107 108 data = {'Video':[yt_img_html_link],109 'Author': [yt_author],110 'Title': [yt_title],111 'Published': [datetime.strptime(yt_publish_date, '%Y-%m-%dT%H:%M:%SZ').strftime('%B %d, %Y')],112 'Views':[format(int(yt_views), ",").replace(",", "'")],113 'Length':[yt_length_isoformat]}114 115 return data, yt_keywords, yt_channel_id116 117 118@st.cache_data()119def get_video_data_from_gsheed(df, video_id):120 121 yt_img_html_link = df.loc[df["ID"] == video_id]['Video'].to_list()[0]122 yt_author = df.loc[df["ID"] == video_id]['Author'].to_list()[0]123 yt_title = df.loc[df["ID"] == video_id]['Title'].to_list()[0]124 yt_publish_date = df.loc[df["ID"] == video_id]['Published'].to_list()[0]125 yt_views = df.loc[df["ID"] == video_id]['Views'].to_list()[0]126 yt_length_isoformat = df.loc[df["ID"] == video_id]['Length'].to_list()[0]127 yt_keywords = df.loc[df["ID"] == video_id]['Keywords'].to_list()[0].split(';')128 yt_channel_id = df.loc[df["ID"] == video_id]['Channel'].to_list()[0]129 130 data = {'Video':[yt_img_html_link],131 'Author': [yt_author],132 'Title': [yt_title],133 'Published': [yt_publish_date],134 'Views':[yt_views],135 'Length':[yt_length_isoformat]}136 137 return data, yt_keywords, yt_channel_id138 139@time_it140def get_punctuated_text(raw_text): 141 response = deeppunkt.predict('sentences',raw_text)142 st.session_state['punkt'] = response143 144 145def get_punctuated_text_to_dict(raw_text):146 #st.session_state['punkt'] = {'data':[raw_text,0,0,0,0], 'duration':0}147 st.session_state['punkt'] = [raw_text,0,0,0,0]148 149 150@time_it151def get_extracted_text(raw_text):152 153 response = lexrank.summarize(raw_text)154 st.session_state['extract'] = response155 156def get_extracted_text_to_dict(raw_text):157 st.session_state['extract'] = [raw_text,0,0,0,0]158 159def get_videos_from_yt(yt):160 161 vids_thumbnails = []162 vids_videoIds = []163 vids_titles = []164 vids_lengths = []165 vids_published= []166 vids_views= []167 item=0168 for video in yt.video_data:169 if item == item_limit:170 break171 item = item+1172 173 vids_video_id = video174 vids_url = 'https://www.youtube.com/watch?v='+vids_video_id175 176 yt_img = f'http://img.youtube.com/vi/{vids_video_id}/mqdefault.jpg'177 yt_img_html = '<img src='+yt_img+' width="250" height="150" />'178 yt_img_html_link = '<a href='+vids_url+'>'+yt_img_html+'</a>'179 vids_thumbnails.append(yt_img_html_link)180 181 vids_video_id_link = '<a target="_self" href="/?vid='+vids_video_id+'">'+vids_video_id+'</a>'182 vids_videoIds.append(vids_video_id_link)183 184 vids_titles.append(yt.video_data[video]['title'])185 186 yt_length = yt.video_data[video]['duration']187 yt_length_isodate = isodate.parse_duration(yt_length)188 yt_length_isoformat = isodate.duration_isoformat(yt_length_isodate, "%H:%M:%S")[1:]189 vids_lengths.append(yt_length_isoformat)190 191 yt_publish_date = yt.video_data[video]['publishedAt']192 yt_publish_date_formatted = datetime.strptime(yt_publish_date, '%Y-%m-%dT%H:%M:%SZ').strftime('%B %d, %Y')193 vids_published.append(yt_publish_date_formatted)194 195 yt_views = yt.video_data[video]['viewCount']196 yt_viws_formatted = format(int(yt_views), ",").replace(",", "'")197 vids_views.append(yt_viws_formatted)198 199 df_videos = {'Video': vids_thumbnails,200 'Video ID':vids_videoIds,201 'Title':vids_titles,202 'Published':vids_published,203 'Views':vids_views,204 'Length':vids_lengths}205 206 return df_videos207 208def get_transcript(video_id): 209 210 # transcript_list = yta.list_transcripts(video_id)211 # # iterate over all available transcripts212 # for transcript in transcript_list:213 # # the Transcript object provides metadata properties214 # st.write(215 # transcript.video_id,216 # transcript.language,217 # transcript.language_code,218 # # whether it has been manually created or generated by YouTube219 # transcript.is_generated,220 # # whether this transcript can be translated or not221 # transcript.is_translatable,222 # # a list of languages the transcript can be translated to223 # transcript.translation_languages,224 # )225 226 transcript_raw = None227 try:228 transcript_list = yta.list_transcripts(video_id)229 transcript_item = transcript_list.find_transcript(['en'])230 except (NoTranscriptFound, TranscriptsDisabled) as e:231 transcript_item = 'No Transcript available.'232 transcript_text = 'No Transcript available.'233 transcript_item_is_generated = False234 return transcript_text, transcript_item_is_generated235 236 transcript_item_is_generated = transcript_item.is_generated237 transcript_raw = transcript_item.fetch()238 239 if transcript_raw is None:240 return None241 242 transcript_text = '\n'.join([i['text'].replace('\n',' ') for i in transcript_raw])243 244 return transcript_text, transcript_item_is_generated245 246def get_meta_info(video_id, url):247 248 lextext = st.session_state.extract[0]249 gpt_sum = '0'250 gpt_title = '0'251 title_sim = '0'252 if len(lextext) < 10:253 gpt_sum = 'NA'254 gpt_title = 'NA'255 title_sim = 'NA'256 257 yt_img = f'http://img.youtube.com/vi/{video_id}/mqdefault.jpg'258 yt_img_html = '<img src='+yt_img+' width="250" height="150" />'259 yt_img_html_link = '<a href='+url+'>'+yt_img_html+'</a>'260 video_info = {'ID': [video_id],261 'Video':[yt_img_html_link],262 'Author': [st.session_state["video_data"]["Author"][0]],263 'Channel':[st.session_state["channel_id"]],264 'Title': [st.session_state["video_data"]["Title"][0]],265 'Published': [st.session_state["video_data"]["Published"][0]],266 'Views':[st.session_state["video_data"]["Views"][0]],267 'Length':[st.session_state["video_data"]["Length"][0]],268 'Keywords':['; '.join(st.session_state["keywords"])]}269 270 transcript_info = {'Words':[int(st.session_state.extract[1])],271 'Sentences': [int(st.session_state.extract[2])],272 'Characters': [int(st.session_state.extract[3])],273 'Tokens':[int(st.session_state.extract[4])],274 'Lextext':[st.session_state.extract[0]],275 'GPTSummary':[gpt_sum],276 'GPTTitle':[gpt_title],277 'Titlesim':[title_sim]}278 df_current_ts = pd.DataFrame({**video_info,**transcript_info})279 280 return df_current_ts281 282 283#######################################################################################284# Application Start 285#######################################################################################286 287 288st.title("Transcriptifier")289st.subheader("Youtube Transcript Downloader")290 291example_urls = [292 'https://www.youtube.com/watch?v=8uQDDUfGNPA', # blog293 'https://www.youtube.com/watch?v=ofZEo0Rzo5s', # h-educate294 'https://www.youtube.com/watch?v=ReHGSGwV4-A', #wholesale ted295 'https://www.youtube.com/watch?v=n8JHnLgodRI', #kevindavid296 'https://www.youtube.com/watch?v=6MI0f6YjJIk', # Nicholas297 'https://www.youtube.com/watch?v=nr4kmlTr9xw', # Linus298 'https://www.youtube.com/watch?v=64Izfm24FKA', # Yannic299 'https://www.youtube.com/watch?v=Mt1P7p9HmkU', # Fogarty300 'https://www.youtube.com/watch?v=bj9snrsSook', #Geldschnurrbart301 'https://www.youtube.com/watch?v=0kJz0q0pvgQ', # fcc302 'https://www.youtube.com/watch?v=gNRGkMeITVU', # iman303 'https://www.youtube.com/watch?v=vAuQuL8dlXo', #ghiorghiu304 'https://www.youtube.com/watch?v=5scEDopRAi0', #infohaus305 'https://www.youtube.com/watch?v=lCnHfTHkhbE', #fcc tutorial306 'https://www.youtube.com/watch?v=QI2okshNv_4'307]308 309 310par_vid = st.experimental_get_query_params().get("vid")311if par_vid:312 par_url = par_vid[0]313else:314 par_url = None315 316select_examples = st.selectbox(label="Choose an example",options=example_urls, key='ex_vid', on_change=update_param_example)317url = st.text_input("Or Enter the YouTube video URL or ID:", value=par_url if par_url else select_examples, key='ti_vid', on_change=update_param_textinput)318 319 320########################321# Load the data for a given video322########################323 324 325API_KEY = st.secrets["api_key"]326yt = YTstats(API_KEY)327#yt = retry_access_yt_object(get_link_from_id(url))328 329if url:330 video_id = get_id_from_link(url)331 332 if 'gsheed' not in st.session_state:333 df = mysheet.read_gspread()334 st.session_state.gsheed = df 335 #st.write("reading spradsheet")336 else:337 df = st.session_state.gsheed338 #st.write("getting spreadsheed from session_state") 339 340 gslist=[]341 try:342 gslist = df.ID.to_list()343 except:344 st.write('no items available.')345 346 if video_id in gslist:347 #st.write(df.loc[df["ID"] == video_id])348 st.write("reading from sheet")349 #transcript_item_is_generated = False350 #transcript_text = df.loc[df["ID"] == video_id]['Punkttext'].to_list()[0]351 #get_punctuated_text_to_dict(transcript_text)352 extracted_text = df.loc[df["ID"] == video_id]['Lextext'].to_list()[0]353 get_extracted_text_to_dict(extracted_text)354 355 video_data, yt_keywords, yt_channel_id = get_video_data_from_gsheed(df, video_id)356 else:357 st.write("reading from api")358 video_data, yt_keywords, yt_channel_id = get_video_data(yt, video_id)359 360 st.session_state["video_data"] = video_data361 st.session_state["keywords"] = yt_keywords362 st.session_state["channel_id"] = yt_channel_id 363 364 365df = pd.DataFrame(st.session_state["video_data"])366st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)367st.write("")368 369###########################370# Load Transcript371###########################372 373transcript_text, transcript_item_is_generated = get_transcript(video_id)374 375#if transcript_text is None:376# st.error("No transcript available.")377# st.stop()378 379########################380# Load Author Keywords, that are not viewable by users381########################382 383keywords_data = {'Authors Keywords':yt_keywords}384st.table(keywords_data)385st.write("")386 387# TODO388# or this video (bj9snrsSook) transcripts are available in the following languages:389 390# (MANUALLY CREATED)391# None392 393# (GENERATED)394# - de ("Deutsch (automatisch erzeugt)")[TRANSLATABLE]395 396# (TRANSLATION LANGUAGES)397# - af ("Afrikaans")398 399 400########################401# Display the transcript along with the download button402########################403 404with st.expander('Preview Transcript'):405 st.code(transcript_text, language=None)406st.download_button('Download Transcript', transcript_text)407 408########################409# API Call to deeppunkt-gr410########################411 412 413st.subheader("Restore Punctuations of Transcript")414if not transcript_item_is_generated:415 st.write("Transcript is punctuated by author.")416 # TODO417 #check if the transcript contains more than 5 sentences418 419if st.button('Load Punctuated Transcript'):420 with st.spinner('Loading Punctuation...'):421 if 'punkt' not in st.session_state:422 # first figure out if transcript is already punctuated423 if transcript_item_is_generated:424 get_punctuated_text(transcript_text)425 else:426 get_punctuated_text_to_dict(transcript_text)427 #st.write('Load time: '+str(round(st.session_state.punkt['duration'],1))+' sec')428 metrics_data = {'Words':[int(st.session_state.punkt[1])],429 'Sentences': [int(st.session_state.punkt[2])],430 'Characters': [int(st.session_state.punkt[3])],431 'Tokens':[int(st.session_state.punkt[4])]}432 df = pd.DataFrame(metrics_data)433 st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)434 st.write("")435 with st.expander('Preview Transcript'):436 st.code(st.session_state.punkt[0], language=None)437 438########################439# Call to lexrank-gr440########################441 442st.subheader("Extract Core Sentences from Transcript")443 444if st.button('Extract Sentences'):445 # decide if the extract is already available, if not, text has to be punctuated first446 with st.spinner('Loading Extractions ...'):447 if 'extract' not in st.session_state:448 with st.spinner('Loading Punctuation for Extraction ...'):449 if 'punkt' not in st.session_state:450 # first figure out if transcript is already punctuated451 if transcript_item_is_generated:452 get_punctuated_text(transcript_text)453 else:454 get_punctuated_text_to_dict(transcript_text)455 456 get_extracted_text(st.session_state.punkt[0])457 458 metrics_data = {'Words':[int(st.session_state.extract[1])],459 'Sentences': [int(st.session_state.extract[2])],460 'Characters': [int(st.session_state.extract[3])],461 'Tokens':[int(st.session_state.extract[4])]}462 463 df = pd.DataFrame(metrics_data)464 st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)465 st.write("")466 467 with st.expander('Preview Transcript'):468 st.code(st.session_state.extract[0], language=None)469 470 ################471 if 'extract' not in st.session_state:472 st.error('Please run extraction first.', icon="🚨")473 else:474 475 df_current_ts = get_meta_info(video_id, url)476 477 # initial write.478 #df_new_sheet = pd.concat([df_current_ts])479 #mysheet.write_gspread(df_new_sheet)480 #st.write(video_info)481 482 if 'gsheed' not in st.session_state:483 df = mysheet.read_gspread()484 st.session_state.gsheed = df485 486 df_sheet = st.session_state.gsheed487 df_current_ts_id = list(df_current_ts.ID)[0]488 if df_current_ts_id not in list(df_sheet.ID):489 df_new_sheet = pd.concat([df_sheet,df_current_ts])490 mysheet.write_gspread(df_new_sheet)491 st.session_state.gsheed = df_new_sheet492 st.write('video added to sheet')493 #else:494 # st.write('video already in sheet')495 # st.write(df_sheet)496 497 498#######################499# write to gspread file500########################501 502if st.button('Read Spreadsheet'):503 504 if 'gsheed' not in st.session_state:505 df = mysheet.read_gspread()506 st.session_state.gsheed = df507 508 st.write(st.session_state.gsheed)509 510 511#if st.button('Add to Spreadsheet'):512 513 514 515 516#######################517# API Call to summarymachine518########################519 520# def get_summarized_text(raw_text):521# response = requests.post("https://wldmr-summarymachine.hf.space/run/predict", json={522# "data": [523# raw_text,524# ]})525# #response_id = response526# if response.status_code == 504:527# raise "Error: Request took too long (>60sec), please try a shorter text."528# return response.json()529 530# st.subheader("Summarize Extracted Sentences with Flan-T5-large")531 532# if st.button('Summarize Sentences'):533# command = 'Summarize the transcript in one sentence:\n\n'534# with st.spinner('Loading Punctuation (Step 1/3)...'):535# if 'punkt' not in st.session_state:536# # first figure out if transcript is already punctuated537# if transcript_item.is_generated:538# get_punctuated_text(transcript_text)539# else:540# get_punctuated_text_to_dict(transcript_text)541# with st.spinner('Loading Extraction (Step 2/3)...'):542# if 'extract' not in st.session_state:543# get_extracted_text(st.session_state.punkt['data'][0])544# with st.spinner('Loading Summary (Step 3/3)...'):545# summary_text = get_summarized_text(command+st.session_state.extract['data'][0])546# st.write('Load time: '+str(round(summary_text['duration'],1))+' sec')547# with st.expander('Preview Transcript'):548# st.write(summary_text['data'][0], language=None)549 550########################551# Channel552########################553 554 555st.subheader("Other Videos of the Channel")556#st.write(st.session_state["channel_id"])557if 'channel_id' not in st.session_state:558 st.error('Channel ID not available.', icon="🚨")559else:560 yt.get_channel_statistics(st.session_state["channel_id"])561 stats_data = {'Channel ID': [st.session_state["channel_id"]],562 'Total Views':[format(int(yt.channel_statistics["viewCount"]), ",").replace(",", "'")],563 'Total Subscribers':[format(int(yt.channel_statistics["subscriberCount"]), ",").replace(",", "'")],564 'Total Videos':[format(int(yt.channel_statistics["videoCount"]), ",").replace(",", "'")],565 }566 df = pd.DataFrame(stats_data)567 st.markdown(df.style.hide(axis="index").to_html(), unsafe_allow_html=True)568st.write("")569 570 571if st.button('Load Videos'):572 573 if 'gsheed' not in st.session_state:574 df = mysheet.read_gspread()575 st.session_state.gsheed = df576 577 progress_text = 'Loading...'578 loading_bar = st.progress(0, text=progress_text)579 item_limit=3580 df = st.session_state.gsheed581 yt.get_channel_video_data(st.session_state["channel_id"],df, loading_bar, progress_text, item_limit)582 583 df_videos = get_videos_from_yt(yt)584 dataset = pd.DataFrame(df_videos)585 st.markdown(dataset.style.hide(axis="index").to_html(), unsafe_allow_html=True)586 587 588########################589# Sequence Loader590########################591 592 593st.subheader("Sequence Loader")594# input hash as secret595 596input_hash = st.text_input("Enter Hash:")597item_limit = st.number_input(label="Number of Videos",value=3)598if st.button('Load Sequence'):599 HASH_KEY = st.secrets["hash_key"]600 if input_hash == HASH_KEY:601 st.write("Access granted")602 # read in spreadsheet603 if 'gsheed' not in st.session_state:604 df = mysheet.read_gspread()605 st.session_state.gsheed = df606 607 progress_text = 'Loading...'608 loading_bar = st.progress(0, text=progress_text)609 df_sheet = st.session_state.gsheed610 yt.get_channel_video_data(st.session_state["channel_id"], df_sheet,loading_bar, progress_text, item_limit)611 df_videos = get_videos_from_yt(yt)612 dataset = pd.DataFrame(df_videos)613 st.markdown(dataset.style.hide(axis="index").to_html(), unsafe_allow_html=True)614 615 for sng in dataset['Video ID']:616 subsng = sng[sng.find('>')+1:sng.find('</')]617 st.write(subsng)618 619 transcript_text, transcript_item_is_generated = get_transcript(subsng)620 621 if transcript_item_is_generated:622 get_punctuated_text(transcript_text)623 else:624 get_punctuated_text_to_dict(transcript_text)625 626 get_extracted_text(st.session_state.punkt[0])627 628 video_data, yt_keywords, yt_channel_id = get_video_data(yt, subsng)629 st.session_state["video_data"] = video_data630 st.session_state["keywords"] = yt_keywords631 st.session_state["channel_id"] = yt_channel_id 632 df_current_ts = get_meta_info(subsng, subsng)633 st.write(df_current_ts)634 df_sheet = st.session_state.gsheed635 df_new_sheet = pd.concat([df_sheet,df_current_ts])636 mysheet.write_gspread(df_new_sheet)637 st.session_state.gsheed = df_new_sheet638 639 st.write('done')640 641 st.write(st.session_state.gsheed)642 643 else:644 st.write("Access denied")645 646 647 648###############649# End of File #650###############651# hide_streamlit_style = """652# <style>653# #MainMenu {visibility: hidden;}654# footer {visibility: hidden;}655# </style>656# """657# st.markdown(hide_streamlit_style, unsafe_allow_html=True)658 659