pikto/GPT3-Dataset-Generator
0
1# ----------------------Importing libraries----------------------2 3import streamlit as st4from streamlit_pills import pills5import pandas as pd6import openai7 8# Imports for AgGrid9from st_aggrid import AgGrid, GridUpdateMode, JsCode10from st_aggrid.grid_options_builder import GridOptionsBuilder11 12# ----------------------Importing utils.py----------------------13 14# For Snowflake (from Tony's utils.py)15import io16from utils import (17 connect_to_snowflake,18 load_data_to_snowflake,19 load_data_to_postgres,20 connect_to_postgres,21)22 23# ----------------------Page config--------------------------------------24 25st.set_page_config(page_title="GPT3 Dataset Generator", page_icon="🤖")26 27# ----------------------Sidebar section--------------------------------28 29# st.image(30# "Gifs/header.gif",31# )32 33st.image("Gifs/boat_new.gif")34 35#API_Key = openai-api-key36 37c30, c31, c32 = st.columns([0.2, 0.1, 3])38 39#################40@st.cache_data # 👈 Add the caching decorator41def load_data(url):42 df = pd.read_csv(url)43 return df44 45#df = load_data("https://github.com/plotly/datasets/raw/master/uber-rides-data1.csv")46#st.dataframe(df)47 48#st.button("Rerun")49 50################51 52with c30:53 54 st.caption("")55 56 st.image("openai.png", width=60)57 58with c32:59 60 st.title("GPT3 Dataset Generator")61 62st.write(63 "This app generates datasets using GPT3. It was created for the ❄️ Snowflake Snowvation Hackathon"64)65 66tabMain, tabInfo, tabTo_dos = st.tabs(["Main", "Info", "To-do's"])67 68with tabInfo:69 st.write("")70 st.write("")71 72 st.subheader("🤖 What is GPT-3?")73 st.markdown(74 "[GPT-3](https://en.wikipedia.org/wiki/GPT-3) is a large language generation model developed by [OpenAI](https://openai.com/) that can generate human-like text. It has a capacity of 175 billion parameters and is trained on a vast dataset of internet text. It can be used for tasks such as language translation, chatbot language generation, and content generation etc."75 )76 77 st.subheader("🎈 What is Streamlit?")78 st.markdown(79 "[Streamlit](https://streamlit.io) is an open-source Python library that allows users to create interactive, web-based data visualization and machine learning applications without the need for extensive web development knowledge"80 )81 82 st.write("---")83 84 st.subheader("📖 Resources")85 st.markdown(86 """87 - OpenAI88 - [OpenAI Playground](https://beta.openai.com/playground)89 - [OpenAI Documentation](https://beta.openai.com/docs) 90 - Streamlit91 - [Documentation](https://docs.streamlit.io/)92 - [Gallery](https://streamlit.io/gallery)93 - [Cheat sheet](https://docs.streamlit.io/library/cheatsheet)94 - [Book](https://www.amazon.com/dp/180056550X) (Getting Started with Streamlit for Data Science)95 - Deploy your apps using [Streamlit Community Cloud](https://streamlit.io/cloud) in just a few clicks 96 """97 )98 99with tabTo_dos:100 101 with st.expander("To-do", expanded=True):102 st.write(103 """104 - [p2] Currently, the results are displayed even if the submit button isn't pressed.105 - [p2] There is still an issue with the index where the first element from the JSON is not being displayed.106 - [Post Hackathon] To limit the number of API calls and costs, let's cap the maximum number - of results to 5. Alternatively, we can consider removing the free API key.107 108 """109 )110 st.write("")111 112 with st.expander("Done", expanded=True):113 st.write(114 """115 - [p2] Check if the Json file is working116 - [p2] On Github, remove any unused images and GIFs.117 - [p1] Add that for postgress - localhost is required118 - [p2] Rename the CSV and JSON as per the st-pills variable119 - [p2] Change the color of the small arrow120 - [p1] Adjust the size of the Gifs121 - Add a streamlit badge in the `ReadMe` file122 - Add the message "Please enter your API key or choose the `Free Key` option."123 - Include a `ReadMe` file124 - Add a section for the Snowflake credentials125 - Remove password from the Python file126 - Add screenshots to the `ReadMe` file127 - Include forms in the snowflake postgres section128 - Remove the hashed code in the Python file129 - Include additional information in the 'info' tab130 - p1] Fix the download issue by sorting it via session state131 - [p1] Make the dataframe from this app editable132 - Add more gifs to the app133 - Change the color scheme to Snowflake Blue134 - Include a section for Snowflake credentials135 - Change the colors of the arrows, using this tool (https://lottiefiles.com/lottie-to-gif/convert)136 - Try new prompts and implement the best ones137 - Add a config file for the color scheme138 - Include an option menu using this tool (https://github.com/victoryhb/streamlit-option-menu)139 - Display a message when the API key is not provided140 - Fix the arrow and rearrange the layout for the API key message141 - Check and improve the quality of the prompt output142 - Send the app to Tony and upload it to GitHub143 - Re-arrange the data on the sidebar144 - Change the colors of both gifs to match the overall color scheme145 - Add context about the app being part of the snowvation project146 - Add a button to convert the data to JSON format147 - Include the Snowflake logo148 - Add a submit button to block API calls unless pressed149 - Add a tab with additional information150 - Resize the columns in the st.form section151 - Add the ability to add the dataset to Snowflake152 - Create a section with pills, showcasing examples153 - Change the main emoji154 - Change the emoji in the tab (page_icon)155 - [INFO] Sort out the issue with credits156 157 158 159 """160 )161 st.write("")162 163 with st.expander("Not needed", expanded=True):164 st.write(165 """166 - Check index issue in readcsv (not an issue as I've changed the script)167 - Add the mouse gif (doesn't fit)168 - Ask Lukas - automatically resize the columns of a DataFrame169 """170 )171 st.write("")172 173 st.write("")174 st.write("")175 st.write("")176 177 178with tabMain:179 180 key_choice = st.sidebar.radio(181 "",182 (183 "Your Key",184 "Free Key (capped)",185 ),186 horizontal=True,187 )188 189 if key_choice == "Your Key":190 191 API_Key = st.sidebar.text_input(192 "First, enter your OpenAI API key", type="password"193 )194 195 elif key_choice == "Free Key (capped)":196 197 API_Key = st.secrets["API_KEY"]198 199 image_arrow = st.sidebar.image(200 "Gifs/blue_grey_arrow.gif",201 )202 203 if key_choice == "Free Key (capped)":204 205 image_arrow.empty()206 207 else:208 209 st.write("")210 211 st.sidebar.caption(212 "No OpenAI API key? Get yours [here!](https://openai.com/blog/api-no-waitlist/)"213 )214 pass215 216 st.write("")217 218 c30, c31, c32 = st.columns([0.2, 0.1, 3])219 220 st.subheader("① Build your dataset")221 222 example = pills(223 "",224 [225 "Sci-fi Movies",226 "Animals",227 "Pop Songs",228 "POTUS's Twitter",229 "Blank",230 ],231 [232 "🍿",233 "🐎",234 "🎵",235 "🇺🇸",236 "👻",237 ],238 label_visibility="collapsed",239 )240 241 if "counter" not in st.session_state:242 st.session_state.counter = 0243 244 def increment():245 st.session_state.counter += 1246 247 if example == "Sci-fi Movies":248 249 with st.form("my_form"):250 251 text_input = st.text_input(252 "What is the topic of your dataset?", value="Sci-fi movies"253 )254 255 col1, col2, col3 = st.columns(3, gap="small")256 257 with col1:258 column_01 = st.text_input("1st column", value="Title")259 260 with col2:261 column_02 = st.text_input("2nd column", value="Year")262 263 with col3:264 column_03 = st.text_input("3rd column", value="PG rating")265 266 col1, col2 = st.columns(2, gap="medium")267 268 with col1:269 number = st.number_input(270 "How many rows do you want?",271 value=5,272 min_value=1,273 max_value=20,274 step=5,275 help="The maximum number of rows is 20.",276 )277 278 with col2:279 engine = st.radio(280 "GPT3 engine",281 (282 "Davinci",283 "Curie",284 "Babbage",285 ),286 horizontal=True,287 help="Davinci is the most powerful engine, but it's also the slowest. Curie is the fastest, but it's also the least powerful. Babbage is somewhere in the middle.",288 )289 290 if engine == "Davinci":291 engine = "davinci-instruct-beta-v3"292 elif engine == "Curie":293 engine = "curie-instruct-beta-v2"294 elif engine == "Babbage":295 engine = "babbage-instruct-beta"296 297 st.write("")298 299 submitted = st.form_submit_button("Build my dataset! ✨", on_click=increment)300 301 elif example == "Animals":302 303 with st.form("my_form"):304 305 text_input = st.text_input(306 "What is the topic of your dataset?", value="Fastest animals on earth"307 )308 309 col1, col2, col3 = st.columns(3, gap="small")310 311 with col1:312 column_01 = st.text_input("1st column", value="Animal")313 314 with col2:315 column_02 = st.text_input("2nd column", value="Speed")316 317 with col3:318 column_03 = st.text_input("3rd column", value="Weight")319 320 col1, col2 = st.columns(2, gap="medium")321 322 with col1:323 number = st.number_input(324 "How many rows do you want?",325 value=5,326 min_value=1,327 max_value=20,328 step=5,329 help="The maximum number of rows is 50.",330 )331 332 with col2:333 engine = st.radio(334 "GPT3 engine",335 (336 "Davinci",337 "Curie",338 "Babbage",339 ),340 horizontal=True,341 help="Davinci is the most powerful engine, but it's also the slowest. Curie is the fastest, but it's also the least powerful. Babbage is somewhere in the middle.",342 )343 344 if engine == "Davinci":345 engine = "davinci-instruct-beta-v3"346 elif engine == "Curie":347 engine = "curie-instruct-beta-v2"348 elif engine == "Babbage":349 engine = "babbage-instruct-beta"350 351 st.write("")352 353 submitted = st.form_submit_button("Build my dataset! ✨", on_click=increment)354 355 elif example == "Stocks":356 357 with st.form("my_form"):358 359 text_input = st.text_input(360 "What is the topic of your dataset?", value="Stocks"361 )362 363 col1, col2, col3 = st.columns(3, gap="small")364 365 with col1:366 column_01 = st.text_input("1st column", value="Ticker")367 368 with col2:369 column_02 = st.text_input("2nd column", value="Price")370 371 with col3:372 column_03 = st.text_input("3rd column", value="Exchange")373 374 col1, col2 = st.columns(2, gap="medium")375 376 with col1:377 number = st.number_input(378 "How many rows do you want?",379 value=5,380 min_value=1,381 max_value=20,382 step=5,383 help="The maximum number of rows is 50.",384 )385 386 with col2:387 engine = st.radio(388 "GPT3 engine",389 (390 "Davinci",391 "Curie",392 "Babbage",393 ),394 horizontal=True,395 help="Davinci is the most powerful engine, but it's also the slowest. Curie is the fastest, but it's also the least powerful. Babbage is somewhere in the middle.",396 )397 398 if engine == "Davinci":399 engine = "davinci-instruct-beta-v3"400 elif engine == "Curie":401 engine = "curie-instruct-beta-v2"402 elif engine == "Babbage":403 engine = "babbage-instruct-beta"404 405 st.write("")406 407 submitted = st.form_submit_button("Build my dataset! ✨", on_click=increment)408 409 elif example == "POTUS's Twitter":410 411 with st.form("my_form"):412 413 text_input = st.text_input(414 "What is the topic of your dataset?", value="POTUS's Twitter accounts"415 )416 417 col1, col2, col3 = st.columns(3, gap="small")418 419 with col1:420 column_01 = st.text_input("1st column", value="Name")421 422 with col2:423 column_02 = st.text_input("2nd column", value="Twitter handle")424 425 with col3:426 column_03 = st.text_input("3rd column", value="# of followers")427 428 col1, col2 = st.columns(2, gap="medium")429 430 with col1:431 number = st.number_input(432 "How many rows do you want?",433 value=5,434 min_value=1,435 max_value=20,436 step=5,437 help="The maximum number of rows is 50.",438 )439 440 with col2:441 engine = st.radio(442 "GPT3 engine",443 (444 "Davinci",445 "Curie",446 "Babbage",447 ),448 horizontal=True,449 help="Davinci is the most powerful engine, but it's also the slowest. Curie is the fastest, but it's also the least powerful. Babbage is somewhere in the middle.",450 )451 452 if engine == "Davinci":453 engine = "davinci-instruct-beta-v3"454 elif engine == "Curie":455 engine = "curie-instruct-beta-v2"456 elif engine == "Babbage":457 engine = "babbage-instruct-beta"458 459 st.write("")460 461 submitted = st.form_submit_button("Build my dataset! ✨")462 463 elif example == "Pop Songs":464 465 with st.form("my_form"):466 467 text_input = st.text_input(468 "What is the topic of your dataset?",469 value="Most famous songs of all time",470 )471 472 col1, col2, col3 = st.columns(3, gap="small")473 474 with col1:475 column_01 = st.text_input("1st column", value="Song")476 477 with col2:478 column_02 = st.text_input("2nd column", value="Artist")479 480 with col3:481 column_03 = st.text_input("3rd column", value="Genre")482 483 col1, col2 = st.columns(2, gap="medium")484 485 with col1:486 number = st.number_input(487 "How many rows do you want?",488 value=5,489 min_value=1,490 max_value=20,491 step=5,492 help="The maximum number of rows is 50.",493 )494 495 with col2:496 engine = st.radio(497 "GPT3 engine",498 (499 "Davinci",500 "Curie",501 "Babbage",502 ),503 horizontal=True,504 help="Davinci is the most powerful engine, but it's also the slowest. Curie is the fastest, but it's also the least powerful. Babbage is somewhere in the middle.",505 )506 507 if engine == "Davinci":508 engine = "davinci-instruct-beta-v3"509 elif engine == "Curie":510 engine = "curie-instruct-beta-v2"511 elif engine == "Babbage":512 engine = "babbage-instruct-beta"513 514 st.write("")515 516 submitted = st.form_submit_button("Build my dataset! ✨")517 518 elif example == "Blank":519 520 with st.form("my_form"):521 522 text_input = st.text_input("What is the topic of your dataset?", value="")523 524 col1, col2, col3 = st.columns(3, gap="small")525 526 with col1:527 column_01 = st.text_input("1st column", value="")528 529 with col2:530 column_02 = st.text_input("2nd column", value="")531 532 with col3:533 column_03 = st.text_input("3rd column", value="")534 535 col1, col2 = st.columns(2, gap="medium")536 537 with col1:538 number = st.number_input(539 "How many rows do you want?",540 value=5,541 min_value=1,542 max_value=20,543 step=5,544 help="The maximum number of rows is 50.",545 )546 547 with col2:548 engine = st.radio(549 "GPT3 engine",550 (551 "Davinci",552 "Curie",553 "Babbage",554 ),555 horizontal=True,556 help="Davinci is the most powerful engine, but it's also the slowest. Curie is the fastest, but it's also the least powerful. Babbage is somewhere in the middle.",557 )558 559 if engine == "Davinci":560 engine = "davinci-instruct-beta-v3"561 elif engine == "Curie":562 engine = "curie-instruct-beta-v2"563 elif engine == "Babbage":564 engine = "babbage-instruct-beta"565 566 st.write("")567 568 submitted = st.form_submit_button("Build my dataset! ✨")569 570 # ----------------------API key section----------------------------------571 572 number = number + 1573 574 if not API_Key and not submitted:575 576 st.stop()577 578 if not API_Key and submitted:579 580 st.info("Please enter your API key or choose the `Free Key` option.")581 st.stop()582 583 if st.session_state.counter >= 100:584 585 pass586 587 # ----------------------API key section----------------------------------588 589 if not submitted and st.session_state.counter == 0:590 591 c30, c31, c32 = st.columns([1, 0.01, 4])592 593 with c30:594 595 st.image("Gifs/arrow_small_new.gif")596 st.caption("")597 598 with c32:599 600 st.caption("")601 st.caption("")602 603 st.info(604 "Enter your dataset's criteria and click the button to generate it."605 )606 607 st.stop()608 609 elif st.session_state.counter > 0:610 611 c30, c31, c32 = st.columns([1, 0.9, 3])612 613 openai.api_key = API_Key614 615 # ----------------------API call section----------------------------------616 617 response = openai.Completion.create(618 model=engine,619 prompt=f"Please provide a list of the top {number} {text_input} along with the following information in a three-column spreadsheet: {column_01}, {column_02}, and {column_03}. The columns should be labeled as follows: {column_01} | {column_02} | {column_03}",620 temperature=0.5,621 max_tokens=1707,622 top_p=1,623 best_of=2,624 frequency_penalty=0,625 presence_penalty=0,626 )627 628 st.write("___")629 630 st.subheader("② Check the results")631 632 with st.expander("See the API Json output"):633 response634 635 output_code = response["choices"][0]["text"]636 637 # ----------------------Dataframe section----------------------------------638 639 # create pandas DataFrame from string640 df = pd.read_csv(io.StringIO(output_code), sep="|")641 # get the number of columns in the dataframe642 num_columns = len(df.columns)643 644 # create a list of column names645 column_names = ["Column {}".format(i) for i in range(1, num_columns + 1)]646 647 # add the header to the dataframe648 df.columns = column_names649 650 # specify the mapping of old column names to new column names651 column_mapping = {652 "Column 1": column_01,653 "Column 2": column_02,654 "Column 3": column_03,655 }656 657 # rename the columns of the dataframe658 df = df.rename(columns=column_mapping)659 660 st.write("")661 662 # ----------------------AgGrid section----------------------------------663 664 gd = GridOptionsBuilder.from_dataframe(df)665 gd.configure_pagination(enabled=True)666 gd.configure_default_column(editable=True, groupable=True)667 gd.configure_selection(selection_mode="multiple")668 gridoptions = gd.build()669 grid_table = AgGrid(670 df,671 gridOptions=gridoptions,672 update_mode=GridUpdateMode.SELECTION_CHANGED,673 theme="material",674 )675 676 # df677 678 # ----------------------Download section--------------------------------------679 680 c30, c31, c32, c33 = st.columns([1, 0.01, 1, 2.5])681 682 with c30:683 684 @st.cache685 def convert_df(df):686 return df.to_csv().encode("utf-8")687 688 csv = convert_df(df)689 690 st.download_button(691 label="Download CSV",692 data=csv,693 file_name=f"{example} dataset .csv",694 mime="text/csv",695 )696 697 with c32:698 699 json_string = df.to_json(orient="records")700 701 st.download_button(702 label="Download JSON",703 data=json_string,704 file_name="data_set_sample.json",705 mime="text/csv",706 )707 708 st.write("___")709 710 st.subheader("③ Load data to Databases")711 712 # Data to load to database(s)713 # df = pd.read_csv("philox-testset-1.csv")714 715 # Get user input for data storage option716 storage_option = st.radio(717 "Select data storage option:",718 (719 "Snowflake",720 "PostgreSQL",721 ),722 horizontal=True,723 )724 725 # Get user input for data storage option726 # Snowflake = st.selectbox(727 # "Select data storage option:", ["Snowflake", "Snowflake"]728 # )729 730 @st.cache(allow_output_mutation=True)731 def reset_form_fields():732 user = ""733 password = ""734 account = ""735 warehouse = ""736 database = ""737 schema = ""738 table = ""739 host = ""740 port = ""741 742 if storage_option == "Snowflake":743 st.subheader("`Enter Snowflake Credentials`👇")744 # Get user input for Snowflake credentials745 746 with st.form("my_form_db"):747 748 col1, col2 = st.columns(2, gap="small")749 750 with col1:751 user = st.text_input("Username:", value="TONY")752 with col2:753 password = st.text_input("Password:", type="password")754 755 with col1:756 account = st.text_input("Account:", value="jn27194.us-east4.gcp")757 with col2:758 warehouse = st.text_input("Warehouse:", value="NAH")759 760 with col1:761 database = st.text_input("Database:", value="SNOWVATION")762 with col2:763 schema = st.text_input("Schema:", value="PUBLIC")764 765 table = st.text_input("Table:")766 767 st.write("")768 769 submitted = st.form_submit_button("Load to Snowflake")770 771 # Load the data to Snowflake772 if submitted:773 # if st.button("Load data to Snowflake"):774 if (775 user776 and password777 and account778 and warehouse779 and database780 and schema781 and table782 ):783 conn = connect_to_snowflake(784 username=user,785 password=password,786 account=account,787 warehouse=warehouse,788 database=database,789 schema=schema,790 )791 if conn:792 load_data_to_snowflake(df, conn, table)793 else:794 st.warning("Please enter all Snowflake credentials")795 796 elif storage_option == "PostgreSQL":797 st.subheader("`Enter PostgreSQL Credentials`👇")798 st.error("Localhost only")799 # Get user input for PostgreSQL credentials800 801 with st.form("my_form_db"):802 803 col1, col2 = st.columns(2, gap="small")804 805 with col1:806 user = st.text_input("Username:", value="postgres")807 with col2:808 password = st.text_input("Password:", type="password")809 with col1:810 host = st.selectbox("Host:", ["localhost", "other"])811 if host == "other":812 host = st.text_input("Enter host:")813 with col2:814 port = st.text_input("Port:", value="5432")815 with col1:816 database = st.text_input("Database:", value="snowvation")817 with col2:818 table = st.text_input("Table:")819 820 st.write("")821 822 submitted = st.form_submit_button("Load to PostgreSQL")823 824 # Load the data to PostgreSQL825 # if st.button("Load data to PostgreSQL"):826 if submitted:827 if user and password and host and port and database and table:828 conn = connect_to_postgres(829 username=user,830 password=password,831 host=host,832 port=port,833 database=database,834 )835 if conn:836 load_data_to_postgres(df, conn, table)837 else:838 st.warning("Please enter all PostgreSQL credentials and table name")839 840 # Reset form fields when storage_option changes841 reset_form_fields()