spunteam/streamlit-web-crawler

1import streamlit as st2from typing import Dict, Any 3import requests4from models import LinkNode, Status5from typing import Dict, Any6import os7from dotenv import load_dotenv8 9load_dotenv()10 11def display_map(link_map: Dict[str, Any]):12    """13    Displays the entire link map in collapsible Streamlit expanders.14    If a link is not relevant based on its overview, it's tagged with a red icon.15    """16    st.header("🌐 Full Exploration Map")17 18    if not link_map:19        st.info("The exploration map is empty.")20        return21 22    validated_map = {}23    for href, dict_node in link_map.items():24        try:25            node = LinkNode.model_validate(dict_node)26            validated_map[href] = node27        except Exception as e:28            st.error(f"Failed to validate data for {href}. Skipping. Error: {e}")29            continue30 31    sorted_map = sorted(validated_map.items(), key=lambda item: item[1].depth)32 33    for href, node in sorted_map:34        st.divider() 35        st.subheader(f"📄 [{href}]({href})")36        if node.parent:37            st.caption(f"Found on: {node.parent}")38 39        status = node.overview.status40        if status == Status.RELEVANT:41            st.success(f"**Status: RELEVANT** ✅")42        elif status == Status.IRRELEVANT:43            st.warning(f"**Status: IRRELEVANT** ⚠️ - Page deemed not relevant to search criteria.")44        elif status == Status.FAILED:45            st.error(f"**Status: FAILED** ❌ - Could not scrape or analyze this page.")46        else:47            st.info(f"**Status: UNKNOWN** 🟡")48 49        st.markdown("**📝 Summary**")50        st.info(node.overview.summary)51 52        with st.expander("View Full Extracted Data and Found Links"):53            st.markdown("##### 📋 Full Extracted Data")54            overview_data = node.overview.model_dump()55            56            display_order = ['details', 'required_docs', 'price', 'SLA']57            58            items_to_display = []59            for key in display_order:60                value = overview_data.get(key)61                if value:62                    title = key.replace('_', ' ').capitalize()63                    items_to_display.append((title, str(value)))64 65            for i, (title, value) in enumerate(items_to_display):66                st.markdown(f"**{title}**")67                st.markdown(value)68                if i < len(items_to_display) - 1: 69                    st.markdown("---")            70            st.markdown("##### 🔗 Links Found on This Page")71            if node.child:72                st.write(f"Found **{len(node.child)}** link(s):")73                links_text = "\n".join(f"- {link}" for link in node.child)74                st.text_area("Links", links_text, height=150, key=f"links_{href}")75            else:76                st.write("No valid links were found on this page.")77 78def main():79    st.title("🤖 Browser Agent: Visa Data Extractor (Streamlit Demo)")80    st.markdown("Enter an API Key and a URL to start a recursive web crawl for structured visa information.")81 82    with st.sidebar:83        st.header("Configuration")84 85        default_url = "https://www.netherlandsworldwide.nl/visa-the-netherlands/visa-application-form"86        87        url = st.text_input("Starting URL (e.g., website.com)", default_url)88 89        max_depth = st.slider("Max Exploration Depth", min_value=1, max_value=5, value=1)90        91        st.markdown("""92        **Note:** Depth 1 is fast. Depth 2 or 3 can be **very slow** and consume many tokens.93        """)94 95    # --- Main Execution ---96    if st.button("Start Exploration and Extraction"):97        print(f"starting crawl for {url} with depth {max_depth}")98        if not url:99            st.error("Please enter a valid Starting URL.")100            return101 102        with st.spinner(f"Crawling {url} up to depth {max_depth}... (This may take a while)"):103            BASE_URI = os.getenv("BASE_URI", "http://localhost:5000")104            print(f"{BASE_URI}/scrape")105            try:106                result = requests.post(107                    f"{BASE_URI}/scrape",108                    headers={"Content-Type": "application/json"},109                    json={110                        "url": url,111                        "max_depth": max_depth112                    }113                )114            except requests.exceptions.ConnectionError:115                 st.error(f"Connection Error: Could not connect to the Flask API at {BASE_URI}. Please ensure your Flask app is running (e.g., `flask run`).")116                 return117            except Exception as e:118                st.exception(f"An unexpected error occurred during the crawl: {e}")119                return120 121        if result.status_code != 200:122            st.error(f"Exploration failed with status {result.status_code}: {result.text}")123            return124            125        data = result.json()126        127        display_map(data.get("link_map", {}))128 129        st.subheader("💰 Accumulated Token Usage (All LLM Calls)")130        token_usage = data.get("token_usage", {"input": 0, "output": 0, "total": 0})131        st.write(f"**Input Tokens:** {token_usage['input']}")132        st.write(f"**Output Tokens:** {token_usage['output']}")133        st.write(f"**Total Tokens:** {token_usage['total']}")134        135if __name__ == "__main__":136    main()137