melrous/CS68_Final_Project_Mo

1import streamlit as st2import os3import requests4from bs4 import BeautifulSoup5from transformers import BartForConditionalGeneration, BartTokenizer6from langchain.document_loaders import UnstructuredURLLoader7 8os.environ["OPENAI_API_KEY"] = st.secrets["api_key"]9 10st.title("🤖MoGPT-Your World Wide Wizard🌎")11url = st.text_input("Enter a URL", placeholder="https://example.com")12question = st.text_input(13    "Ask something about the web page",14    placeholder="What did the company say about performance?",15    disabled=not url,16)17 18if st.button("Submit question"):19    if not url.startswith("http"):20        st.write("Please enter a valid URL starting with 'http' or 'https'.")21    else:22        # Fetch web page content using langchain UnstructuredURLLoader23        try:24            loader = UnstructuredURLLoader(urls=[url], ssl_verify=False, headers={"User-Agent": "Mozilla/5.0"})25            documents = loader.load()26 27            # Extract text content from each Document object using requests and BeautifulSoup28            text = ""29            for document in documents:30                response = requests.get(url)31                response.raise_for_status()32                soup = BeautifulSoup(response.content, "html.parser")33                text += " ".join(item.get_text() for item in soup.find_all(["p", "h1", "h2", "h3", "h4", "h5", "h6"]))34 35        except Exception as e:36            st.write(f"Error fetching content from URL: {e}")37            text = None38 39        if text:40            # Process text and perform question-answering41            model = BartForConditionalGeneration.from_pretrained("facebook/bart-large-cnn")42 43            # Tokenize the question and text using the Hugging Face tokenizer44            tokenizer = BartTokenizer.from_pretrained("facebook/bart-large-cnn")45            inputs = tokenizer(question, text, return_tensors="pt", max_length=1024, truncation=True)46 47            # Generate the answer using the model48            answer_ids = model.generate(inputs.input_ids, attention_mask=inputs.attention_mask, max_length=300)49            answer = tokenizer.decode(answer_ids[0], skip_special_tokens=True)50 51            st.write("### Answer")52            st.write(answer)53