ositamiles/Spider-crawler

1import streamlit as st2import pandas as pd3import os4import tempfile5from scrapy import IGDBSpider  # Make sure to use the correct spider name6from scrapy.crawler import CrawlerRunner7from twisted.internet import reactor, defer8from scrapy.utils.log import configure_logging9 10# Function to run the Scrapy spider and store data in a temporary CSV file11@st.cache_data12def run_scrapy_spider():13    # Disable Scrapy's default log handling14    configure_logging()15    16    with tempfile.NamedTemporaryFile(delete=False, suffix=".csv") as temp_file:17        temp_file_path = temp_file.name18        19        # CrawlerRunner does not handle signals, avoiding the 'EPollReactor' issue20        runner = CrawlerRunner(settings={21            'FEED_FORMAT': 'csv',22            'FEED_URI': temp_file_path23        })24 25        @defer.inlineCallbacks26        def crawl():27            yield runner.crawl(IGDBSpider)28            reactor.stop()29 30        # Start the reactor manually and crawl the website31        reactor.callWhenRunning(crawl)32        reactor.run()  # Blocking call until spider completes33 34        return temp_file_path  # Return the temporary file path35 36# Load scraped CSV data37def load_data(file_path):38    if os.path.exists(file_path) and os.path.getsize(file_path) > 0:39        return pd.read_csv(file_path)40    else:41        return None42 43# Streamlit app layout44st.title("B2B Game Marketplace - Recently Released Games Scraping")45 46st.write("""47This application scrapes recently released games from IGDB and converts the data into a CSV dataset for the B2B game marketplace.48""")49 50if st.button('Run Scraping'):51    with st.spinner('Scraping recently released games...'):52        file_path = run_scrapy_spider()53        st.success('Scraping completed!')54 55        # Display scraped game data56        data = load_data(file_path)57        if data is not None and not data.empty:58            st.write("### Scraped Game Data", data.head())59 60            # Convert to CSV for download61            csv = data.to_csv(index=False)62            st.download_button(63                label="Download Game Data as CSV",64                data=csv,65                file_name='recent_games.csv',66                mime='text/csv',67            )68        else:69            st.info('No data available. Please run the scraping again.')70else:71    st.info('Please click the button to start scraping.')