Shad0ws/website-to-knowledge-base

1import streamlit as st2import pandas as pd3 4from knowledge_base import KnowledgeBase5 6# Page setup7st.set_page_config(page_title="Website to AI-Powered Knowledge Base", page_icon="🐍")8st.title("AI-Powered Knowledge Base")9 10# Remove whitespace from the top of the page and sidebar11st.markdown(12    """13        <style>14               .css-18e3th9 {15                    padding-top: 0rem;16                    padding-bottom: 10rem;17                    padding-left: 5rem;18                    padding-right: 5rem;19                }20               .css-1d391kg {21                    padding-top: 3.5rem;22                    padding-right: 1rem;23                    padding-bottom: 3.5rem;24                    padding-left: 1rem;25                }26        </style>27""",28    unsafe_allow_html=True,29)30 31st.markdown("## Config")32 33col1, col2 = st.columns(2)34 35with col1:36    sitemap_url = st.text_input("URL to the website sitemap", value="")37 38with col2:39    pattern = st.text_input("URL filter pattern (optional)", value="")40 41 42st.markdown("## Ask")43 44 45@st.cache_resource46def get_knowledge_base(url, pattern):47    return KnowledgeBase(48        sitemap_url=url,49        pattern=pattern,50        chunk_size=8000,51        chunk_overlap=3000,52    )53 54 55@st.cache_resource56def get_answer(url, pattern, query):57    kb = get_knowledge_base(sitemap_url, pattern)58    return kb.ask(query)59 60 61if sitemap_url and pattern:62    with st.spinner("Getting the knowledge base ready, this may take a bit ..."):63        kb = get_knowledge_base(sitemap_url, pattern)64 65    query = st.text_input("Question", value="")66 67    if query:68        with st.spinner("Getting the answer ..."):69            result = get_answer(sitemap_url, pattern, query)70 71        st.markdown("### Answer")72        st.markdown(result["answer"])73        st.markdown("### Sources")74        st.markdown("\n ".join([f"- {x}" for x in result["sources"].split("\n")]))75