Metafazer/finrag-backend

1"""CLI script to batch download SEC filings (10-K and 10-Q) for AAPL, MSFT, TSLA, GOOG, AMZN.2 3Usage:4    python scripts/ingest_all.py5"""6 7import asyncio8import sys9from pathlib import Path10 11# Add src to path for direct script execution12sys.path.insert(0, str(Path(__file__).parent.parent / "src"))13 14import structlog15 16from finrag.config import get_settings17from finrag.ingestion.edgar_client import (18    EdgarError,19    ingest_filing,20)21 22structlog.configure(23    processors=[24        structlog.processors.TimeStamper(fmt="iso"),25        structlog.processors.add_log_level,26        structlog.dev.ConsoleRenderer(),27    ],28)29 30logger = structlog.get_logger(__name__)31 32# List of tickers to download33TICKERS = ["AAPL", "MSFT", "TSLA", "GOOG", "AMZN"]34FILING_TYPES = ["10-K", "10-Q"]35COUNT = 236 37async def run_batch_ingestion() -> None:38    """Download filings for all configured tickers and types sequentially to respect rate limits."""39    settings = get_settings()40    total_saved = 041 42    logger.info("batch_ingestion_started", tickers=TICKERS, filing_types=FILING_TYPES, count_per_type=COUNT)43 44    for ticker in TICKERS:45        for filing_type in FILING_TYPES:46            logger.info("ingesting_ticker_filings", ticker=ticker, filing_type=filing_type)47            try:48                # Ingest filings49                saved_paths = await ingest_filing(50                    ticker=ticker,51                    filing_type=filing_type,52                    settings=settings,53                    count=COUNT,54                )55                56                for path in saved_paths:57                    logger.info("filing_saved", ticker=ticker, filing_type=filing_type, path=str(path))58                59                total_saved += len(saved_paths)60                61                # Small polite sleep between request batches to avoid triggering EDGAR rate limits62                await asyncio.sleep(2)63                64            except EdgarError as e:65                logger.error("ingestion_failed_for_item", ticker=ticker, filing_type=filing_type, error=str(e))66            except Exception as e:67                logger.error("unexpected_error", ticker=ticker, filing_type=filing_type, error=str(e))68 69    logger.info("batch_ingestion_complete", total_directories_saved=total_saved)70 71def main() -> None:72    try:73        asyncio.run(run_batch_ingestion())74    except KeyboardInterrupt:75        logger.info("ingestion_interrupted")76        sys.exit(130)77 78if __name__ == "__main__":79    main()80