sridattapradeep/India_Equity_Scanner
0
1"""2main.py — FastAPI application for Indian Equity Scanner3 4Architecture:5 - Background scheduler runs a full scan every 30 minutes and on startup.6 - All four scan endpoints serve from the SQLite cache (<100ms response).7 - POST /api/scan/trigger starts an immediate background scan.8 - GET /api/scan/status reports whether a scan is running + last run metadata.9 - Live fallback: if the cache is empty (first ever start), endpoints run live.10"""11from __future__ import annotations12 13import logging14import os15import threading16from types import SimpleNamespace17import time18from concurrent.futures import ThreadPoolExecutor, as_completed19from contextlib import asynccontextmanager20from datetime import datetime, timedelta, timezone21from typing import Optional22 23import numpy as np24import pandas as pd25import yfinance as yf26from apscheduler.schedulers.background import BackgroundScheduler27from cachetools import TTLCache28from dotenv import load_dotenv29from fastapi import Depends, FastAPI, HTTPException, Query, Request30from fastapi.middleware.cors import CORSMiddleware31from fastapi.responses import JSONResponse, Response32from pydantic import BaseModel33from sqlalchemy import create_engine34from sqlalchemy.orm import Session, sessionmaker35 36from models import (37 Base, init_db,38 ScanRun,39 VerdictCase,40 MomentumResult as MomentumResultORM,41 SwingSetup as SwingSetupORM,42 SMCSetup as SMCSetupORM,43 ReversalWatchlist as ReversalWatchlistORM,44 MarketSentiment as MarketSentimentORM,45 NewsItem as NewsItemORM,46)47from scanner import (48 FETCH_DELAY_SEC,49 NIFTY_500_UNIVERSE,50 MomentumData,51 _NSE_ARCHIVE_URL,52 _NSE_EQUITY_MASTER_URL,53 _NSE_HEADERS,54 _bars_since_swing_qualify,55 _compute_indicators,56 _fetch_ohlcv,57 _fetch_ohlcv_batch,58 compute_breadth_history,59 refresh_universe,60 scan_momentum,61 scan_reversal_watchlist,62 scan_swing_setups,63)64from scanner import BENCHMARK_SYMBOL65from smc_engine import analyse_stock66import backtest as backtest_mod67import email_digest68import excel_export69import auth70import upstox_client71import rrg72import verdict73import paper_trading74import decision_log75 76# ── Environment & logging ────────────────────────────────────────────────────77load_dotenv()78 79logging.basicConfig(80 level=logging.INFO,81 format="%(asctime)s | %(levelname)s | %(message)s",82)83logger = logging.getLogger(__name__)84 85# ── Database ─────────────────────────────────────────────────────────────────86DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./scanner.db")87if DATABASE_URL.startswith("sqlite"):88 _connect_args: dict = {"check_same_thread": False}89 _pool_kwargs: dict = {}90else:91 # Same Neon-pooler tuning as auth.py's auth_engine (see the comment there,92 # commit around 2026-06-23): PgBouncer transaction mode rejects93 # statement_timeout etc. as a startup parameter (crash-loops the Space on94 # boot), so those are set via SET on each new connection instead; TCP95 # keepalives make a dead Neon connection surface in ~15s instead of96 # hanging for minutes.97 _connect_args = {98 "connect_timeout": 10,99 "keepalives": 1,100 "keepalives_idle": 5,101 "keepalives_interval": 5,102 "keepalives_count": 3,103 }104 _pool_kwargs = {"pool_recycle": 280}105engine = create_engine(DATABASE_URL, connect_args=_connect_args, pool_pre_ping=True, **_pool_kwargs)106SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)107 108if not DATABASE_URL.startswith("sqlite"):109 from sqlalchemy import event as _sa_event110 111 @_sa_event.listens_for(engine, "connect")112 def _set_scan_db_timeouts(dbapi_connection, _record):113 cursor = dbapi_connection.cursor()114 cursor.execute("SET statement_timeout = 15000")115 cursor.execute("SET lock_timeout = 10000")116 cursor.close()117 118# ── In-memory cache: market sentiment (3 min) + others ──────────────────────119_sentiment_cache: TTLCache = TTLCache(maxsize=1, ttl=180) # 3 min — fast_info prices120_sector_cache: TTLCache = TTLCache(maxsize=1, ttl=900)121_stock_cache: TTLCache = TTLCache(maxsize=50, ttl=120) # per-symbol stock detail, 2-min TTL122_stocks_list_cache: TTLCache = TTLCache(maxsize=1, ttl=3600) # autocomplete list, 1-hour TTL123_chart_cache: TTLCache = TTLCache(maxsize=200, ttl=1800) # OHLCV chart bars, 30-min TTL124_live_px_cache: TTLCache = TTLCache(maxsize=600, ttl=120) # batch + per-symbol live prices, 2-min TTL125_stock_bt_cache: TTLCache = TTLCache(maxsize=100, ttl=1800) # per-symbol backtests, 30-min TTL126_rrg_cache: TTLCache = TTLCache(maxsize=16, ttl=1800) # RRG overview + per-sector analysis, 30-min TTL127 128# cachetools.TTLCache is NOT thread-safe, but every cache above is read/written129# from concurrent request threads plus the background scan thread. Guard all130# access with one lock. The lock only protects cache integrity — it is never131# held during the expensive fetch, so a cache miss doesn't serialise requests.132_cache_lock = threading.Lock()133 134def _cache_get(cache: TTLCache, key):135 """Thread-safe get; returns None when absent/expired (cached values are never None)."""136 with _cache_lock:137 try:138 return cache[key]139 except KeyError:140 return None141 142def _cache_set(cache: TTLCache, key, value) -> None:143 """Thread-safe set."""144 with _cache_lock:145 cache[key] = value146 147def _cache_clear(cache: TTLCache) -> None:148 """Thread-safe clear."""149 with _cache_lock:150 cache.clear()151 152 153# NSE sector index symbols (Yahoo Finance)154SECTOR_INDICES = [155 {"name": "IT", "symbol": "^CNXIT"},156 {"name": "Bank", "symbol": "^NSEBANK"},157 {"name": "FMCG", "symbol": "^CNXFMCG"},158 {"name": "Auto", "symbol": "^CNXAUTO"},159 {"name": "Pharma", "symbol": "^CNXPHARMA"},160 {"name": "Metal", "symbol": "^CNXMETAL"},161 {"name": "Energy", "symbol": "^CNXENERGY"},162 {"name": "Realty", "symbol": "^CNXREALTY"},163 {"name": "Infra", "symbol": "^CNXINFRA"},164 {"name": "PSE", "symbol": "^CNXPSE"},165 {"name": "PSU Bank", "symbol": "^CNXPSUBANK"},166 {"name": "Media", "symbol": "^CNXMEDIA"},167]168 169# Indices Heatmap (Phase 2c). Maps an indices_nse classification key → the Yahoo170# Finance index ticker that carries that index's REAL published level. These are171# the "official" tiles. Every other tracked index has no reliable Yahoo feed, so172# it falls back to an equal-weight constituent proxy (is_proxy=True) computed each173# scan in _compute_index_proxies(). If an official ticker returns no data the tile174# also degrades gracefully to its proxy.175OFFICIAL_INDEX_TICKERS = {176 # Broad market177 "nifty50": "^NSEI",178 "nifty100": "^CNX100",179 "nifty500": "^CRSLDX",180 "midcap50": "^NSEMDCP50",181 # Sectoral182 "bank": "^NSEBANK",183 "it": "^CNXIT",184 "fmcg": "^CNXFMCG",185 "auto": "^CNXAUTO",186 "pharma": "^CNXPHARMA",187 "metal": "^CNXMETAL",188 "realty": "^CNXREALTY",189 "psubank": "^CNXPSUBANK",190 "media": "^CNXMEDIA",191 # Thematic192 "energy": "^CNXENERGY",193 "infra": "^CNXINFRA",194 "pse": "^CNXPSE",195}196 197# ── Background scan state ─────────────────────────────────────────────────────198_is_scanning = False199_scan_lock = threading.Lock()200 201# Market-breadth history (Phase 8). Recomputed from the universe price window on202# every scan and held in memory — see compute_breadth_history(). Guarded by203# _cache_lock. A plain list assignment is atomic in CPython, but the lock keeps204# readers from observing a torn update and documents the shared-state contract.205_breadth_history: list[dict] = []206_breadth_meta: dict = {"computed_at": None}207 208# Per-stock snapshot for the Browse page (Phase 2). {bare_symbol: {change_pct,209# close}} recomputed every scan from the full-universe momentum data — free, no210# extra fetch. Held in memory like breadth; guarded by _cache_lock.211_universe_snapshot: dict = {}212 213# Equal-weight constituent proxy returns per index (Phase 2c) for the Indices214# Heatmap. {index_key: {change_1d, change_5d, change_1m, change_3m, count}}.215# Recomputed every scan from the shared 18mo price window (free) and used for216# every index that lacks an OFFICIAL_INDEX_TICKERS feed. Guarded by _cache_lock.217_index_proxy: dict = {}218 219# Smart Money snapshot (Phase 3) — bulk/block deals, FII/DII flows, corporate220# actions & announcements from NSE's dynamic API (see nse_live.py). Refreshed221# once per trading day (startup + daily cron), NOT per scan. Guarded by222# _cache_lock. Deals are cross-referenced against the latest scan's setups.223_smart_money: dict = {}224 225# {bare_symbol: [screener names]} from the latest completed scan — used to226# annotate Smart Money deals. Published into memory once per scan (and on the227# daily smart-money refresh) so the /api/smart-money endpoint never queries228# SQLite on the hot path; this keeps it fast and avoids read/lock contention229# with the scan's own DB writes. Guarded by _cache_lock.230_setup_symbols: dict = {}231 232# {bare_symbol: fundamentals_score} (Phase 13d) — published once per daily233# fundamentals batch so the min_fundamentals_score confluence filter on the234# 4 screener endpoints + /api/browse never hits the (separate, Postgres)235# fundamentals DB on the hot path. Guarded by _cache_lock.236_fundamentals_score_cache: dict = {}237 238# {bare_symbol: {score, growth_score, quality_score, valuation_score,239# promoter_pct, promoter_trend, data_freshness_days}} — richer sibling of240# _fundamentals_score_cache, populated in the same refresh pass, for the241# Fundamentals screener page (needs sub-scores, not just the composite).242# Guarded by _cache_lock.243_fundamentals_full_cache: dict = {}244 245# Walk-forward screener backtest (Phase 12). Pure function of the price246# window, like breadth — recomputed once per trading day in a background247# thread (it fetches its own 3-year batch) and held in memory, so it survives248# redeploys without a persistent DB. Guarded by _cache_lock; _backtest_lock249# prevents two computations from running at once.250_backtest_result: dict = {}251_backtest_meta: dict = {"computed_at": None, "data_date": None, "running": False}252_backtest_lock = threading.Lock()253 254 255def _compute_backtest(data_date: str) -> None:256 """Fetch a 3-year window and run the walk-forward backtest. Runs in its257 own daemon thread so it never delays the 30-minute scan loop."""258 global _backtest_result, _backtest_meta259 if not _backtest_lock.acquire(blocking=False):260 return261 try:262 with _cache_lock:263 _backtest_meta["running"] = True264 logger.info("[backtest] computing for data date %s (3y fetch)…", data_date)265 t0 = time.perf_counter()266 dfs = _fetch_ohlcv_batch(NIFTY_500_UNIVERSE, period="3y")267 bench = _fetch_ohlcv(BENCHMARK_SYMBOL, period="3y")268 result = backtest_mod.run_backtest(dfs, bench)269 with _cache_lock:270 _backtest_result = result271 _backtest_meta = {272 "computed_at": datetime.now(timezone.utc).isoformat(),273 "data_date": data_date,274 "running": False,275 "duration_sec": round(time.perf_counter() - t0, 1),276 }277 logger.info("[backtest] done in %.1fs — status=%s",278 time.perf_counter() - t0, result.get("status"))279 except Exception as exc:280 logger.error("[backtest] failed: %s", exc, exc_info=True)281 with _cache_lock:282 _backtest_meta["running"] = False283 finally:284 _backtest_lock.release()285 286 287def _maybe_schedule_backtest(universe_dfs: dict) -> None:288 """Kick off a backtest run if the latest bar date moved past the last289 computed one (i.e. at most once per trading day, plus after deploys)."""290 try:291 last_dates = [df.index[-1] for df in universe_dfs.values() if len(df)]292 if not last_dates:293 return294 data_date = max(last_dates).strftime("%Y-%m-%d")295 with _cache_lock:296 already = _backtest_meta.get("data_date") == data_date297 running = _backtest_meta.get("running", False)298 if already or running:299 return300 threading.Thread(target=_compute_backtest, args=(data_date,),301 daemon=True, name="backtest").start()302 except Exception as exc:303 logger.warning("[backtest] scheduling failed (non-fatal): %s", exc)304 305 306def _compute_index_proxies(universe_dfs: dict, membership: dict) -> dict:307 """Equal-weight constituent average return per index over 1D/5D/1M/3M,308 computed from the shared 18-month price window. Free (no extra fetch) — used309 as the proxy series on the Indices Heatmap for every index without a live310 Yahoo feed. Returns {index_key: {change_1d, change_5d, change_1m, change_3m,311 count}}. A period is null when too few sessions exist."""312 spans = {"change_1d": 1, "change_5d": 5, "change_1m": 21, "change_3m": 63}313 314 # Per-symbol % returns over each span, computed once. _fetch_ohlcv_batch315 # returns lowercase columns ("close"); be tolerant of either case.316 sym_returns: dict[str, dict] = {}317 for sym_ns, df in universe_dfs.items():318 try:319 col = "close" if "close" in df.columns else "Close"320 close = df[col].dropna()321 except Exception:322 continue323 if len(close) < 2:324 continue325 last = float(close.iloc[-1])326 if last <= 0:327 continue328 r = {}329 for label, n in spans.items():330 if len(close) > n:331 base = float(close.iloc[-n - 1])332 r[label] = ((last - base) / base * 100) if base else None333 else:334 r[label] = None335 sym_returns[sym_ns.replace(".NS", "")] = r336 337 out: dict[str, dict] = {}338 for key, syms in membership.items():339 agg = {label: [] for label in spans}340 for s in syms:341 r = sym_returns.get(s)342 if not r:343 continue344 for label in spans:345 if r[label] is not None:346 agg[label].append(r[label])347 tile = {label: (round(sum(v) / len(v), 2) if v else None) for label, v in agg.items()}348 tile["count"] = max((len(v) for v in agg.values()), default=0)349 out[key] = tile350 return out351 352 353# ============================================================354# BACKGROUND SCAN — core logic, runs in a daemon thread355# ============================================================356 357def _startup_build_and_scan() -> None:358 """359 Startup background task: expand the universe to the full liquidity-filtered360 NSE list (nsepython, ~1100-1200 stocks) BEFORE the first scan, then scan.361 The import-time universe is the fast archive CSV (~504); this swaps in the362 larger one without blocking app boot / the HF healthcheck. Falls back to the363 import-time universe if the build fails.364 """365 try:366 n = refresh_universe()367 if n:368 _cache_clear(_stocks_list_cache)369 logger.info("[startup] universe ready: %d stocks", n or len(NIFTY_500_UNIVERSE))370 except Exception as exc:371 logger.warning("[startup] universe build failed (%s) — using import-time universe", exc)372 _build_classification()373 _build_upstox_instruments()374 _build_upstox_index_instruments()375 _execute_full_scan()376 _refresh_smart_money()377 _refresh_fundamentals_score_cache()378 379 380def _startup_news_poll() -> None:381 """Runs in its OWN thread (see lifespan) rather than at the tail of382 _startup_build_and_scan — that scan takes ~10-20 min on a fresh universe383 build, and News Monitor is supposed to populate within seconds of boot,384 not wait behind it. Resilient: any failure is logged, never fatal."""385 _poll_news_macro()386 _refresh_macro_indicators()387 try:388 import news_monitor389 db = SessionLocal()390 try:391 news_monitor.poll_nse_announcements(db) # ungated once, so boot never starts empty392 finally:393 db.close()394 except Exception as exc:395 logger.warning("[startup] news announcements poll failed (non-fatal): %s", exc)396 397 398def _build_classification() -> None:399 """Fetch NSE index constituent CSVs → industry + index-membership maps for400 the Browse page. Resilient: any failure leaves the last good classification401 (in-memory or classification.json) in place. Runs at startup + daily cron."""402 try:403 import indices_nse404 cls = indices_nse.build_classification()405 logger.info(406 "[startup] classification ready: %d indices, %d industry tags",407 len(cls.get("indices", [])), len(cls.get("industries", {})),408 )409 except Exception as exc:410 logger.warning("[startup] classification build failed: %s", exc)411 412 413def _build_upstox_instruments() -> None:414 """Fetch Upstox's NSE instrument master → symbol->instrument_key map used415 by _fast_info_batch/_fast_price for live LTP quotes. Resilient: any failure416 (no token, network) leaves the last good map (in-memory or417 upstox_instruments.json) in place — callers just fall back to yfinance.418 Runs at startup + daily cron."""419 try:420 mapping = upstox_client.build_instrument_map()421 logger.info("[startup] upstox instrument map ready: %d symbols", len(mapping))422 except Exception as exc:423 logger.warning("[startup] upstox instrument map build failed: %s", exc)424 425 426def _build_upstox_index_instruments() -> None:427 """Fetch Upstox's NSE_INDEX instrument map (RRG feature) → name->instrument_key428 used by rrg.fetch_price_history() for weekly sector-index candles.429 Resilient: any failure leaves the last good map in place (in-memory or430 upstox_index_instruments.json); rrg.py falls back to yfinance per-index if431 a name never resolves. Runs at startup + daily cron."""432 try:433 mapping = upstox_client.build_index_instrument_map()434 logger.info("[startup] upstox index instrument map ready: %d indices", len(mapping))435 except Exception as exc:436 logger.warning("[startup] upstox index instrument map build failed: %s", exc)437 438 439def _refresh_smart_money() -> None:440 """Fetch the NSE Smart Money snapshot (bulk/block deals, FII/DII, corp441 actions & announcements) → in-memory + smart_money.json. Resilient: any442 failure leaves the last good snapshot in place. Runs at startup + daily."""443 try:444 import nse_live445 snap = nse_live.build_snapshot()446 global _smart_money447 with _cache_lock:448 _smart_money = snap449 logger.info(450 "[smart-money] ready: fii_dii=%d bulk=%d block=%d corp_actions=%d announce=%d",451 len(snap.get("fii_dii", [])), len(snap.get("bulk_deals", [])),452 len(snap.get("block_deals", [])), len(snap.get("corp_actions", [])),453 len(snap.get("announcements", [])),454 )455 except Exception as exc:456 logger.warning("[smart-money] refresh failed (non-fatal): %s", exc)457 458 459def _in_nse_market_hours() -> bool:460 """True Mon-Fri 9:00-16:00 IST (a few minutes either side of the 9:15-15:30461 session so the open/close prints aren't missed). Doesn't account for NSE462 holidays — worst case a handful of wasted polls on a holiday, harmless."""463 from datetime import timedelta as _td464 now_ist = datetime.now(timezone.utc) + _td(hours=5, minutes=30)465 return now_ist.weekday() < 5 and 9 <= now_ist.hour < 16466 467 468def _poll_news_announcements() -> None:469 """Fast-cadence job (News Monitor): NSE corporate announcements, the470 highest-value/most time-sensitive feed — filed by companies throughout471 the trading day. Only polls during market hours; resilient to any472 failure (network, NSE bot-gate re-prime, DB)."""473 if not _in_nse_market_hours():474 return475 try:476 import news_monitor477 db = SessionLocal()478 try:479 news_monitor.poll_nse_announcements(db)480 finally:481 db.close()482 except Exception as exc:483 logger.warning("[news-monitor] announcements poll failed (non-fatal): %s", exc)484 485 486def _poll_news_macro() -> None:487 """Medium-cadence job (News Monitor): NSE corporate actions, RBI press488 releases, and broad macro/market news (Google News RSS). Runs any time489 (not gated to market hours) since none of these are confined to the490 trading session. Resilient to any failure."""491 try:492 import news_monitor493 db = SessionLocal()494 try:495 news_monitor.poll_nse_corp_actions(db)496 news_monitor.poll_rbi(db)497 news_monitor.poll_macro_news(db)498 finally:499 db.close()500 except Exception as exc:501 logger.warning("[news-monitor] macro poll failed (non-fatal): %s", exc)502 503 504def _refresh_macro_indicators() -> None:505 """News Monitor's Macro Indicators panel: RBI policy rates, GDP growth506 (FRED), best-effort CPI/WPI (from ingested news). These change rarely507 (rates only after MPC meetings, GDP quarterly, CPI/WPI monthly) so a508 daily refresh is plenty — runs at boot (own thread) + daily cron.509 Resilient: any failure leaves the last good snapshot in place."""510 try:511 import macro_indicators512 db = SessionLocal()513 try:514 macro_indicators.build_macro_snapshot(db)515 finally:516 db.close()517 except Exception as exc:518 logger.warning("[macro] refresh failed (non-fatal): %s", exc)519 520 521def _refresh_fundamentals_batch(n: Optional[int] = None) -> None:522 """Daily cron target (Phase 13a): fetch the next ~90-symbol slice of NSE523 quarterly results + the full promoter-holdings snapshot, then republish524 the in-memory fundamentals-score cache (Phase 13d) so the confluence525 filters reflect today's batch. `n` overrides the batch size for a one-off526 manual soak test (see /api/debug/fundamentals-test-batch) — the daily527 cron itself always calls this with n=None (the permanent default).528 Resilient: any failure is logged, never fatal to the scan loop."""529 try:530 import fundamentals_nse531 summary = fundamentals_nse.run_fundamentals_batch(NIFTY_500_UNIVERSE, n=n)532 logger.info("[fundamentals] %s", summary)533 except Exception as exc:534 logger.warning("[fundamentals] batch refresh failed (non-fatal): %s", exc)535 _refresh_fundamentals_score_cache()536 537 538def _refresh_fundamentals_score_cache() -> None:539 """Compute every tracked symbol's fundamentals score and publish it to540 memory — avoids a per-request round-trip to the (separate, Postgres)541 fundamentals DB when the 4 screener endpoints / Browse apply a542 min_fundamentals_score filter. Resilient: any failure leaves the last543 good cache in place."""544 try:545 import fundamentals_nse as fn546 session = fn.AuthSession()547 try:548 history_by_symbol: dict[str, list[dict]] = {}549 for r in session.query(fn.FundamentalsSnapshot).order_by(fn.FundamentalsSnapshot.period_to.desc()).all():550 history_by_symbol.setdefault(r.symbol, []).append({551 "revenue": r.revenue, "net_profit": r.net_profit,552 "eps": r.eps, "debt_equity": r.debt_equity,553 })554 promoter_by_symbol: dict[str, list[dict]] = {}555 for p in session.query(fn.PromoterHolding).order_by(fn.PromoterHolding.as_of_date.desc()).all():556 promoter_by_symbol.setdefault(p.symbol, []).append({"promoter_pct": p.promoter_pct})557 558 cursor_by_symbol: dict[str, datetime] = {559 c.symbol: c.last_fetched_at560 for c in session.query(fn.FundamentalsCursor).all()561 if c.last_fetched_at562 }563 564 universe_pes = _universe_pe_sample(session, fn.FundamentalsSnapshot)565 566 now = datetime.now(timezone.utc)567 cache = {}568 full_cache = {}569 for sym, history in history_by_symbol.items():570 live_price = _universe_snapshot.get(sym, {}).get("close")571 eps = history[0].get("eps") if history else None572 pe = (live_price / eps) if (live_price and eps and eps > 0) else None573 pe_percentile = fn.pe_percentile_in_universe(pe, universe_pes)574 promoter_hist = promoter_by_symbol.get(sym, [])575 score = fn.compute_fundamentals_score(history, promoter_hist, pe_percentile)576 cache[sym] = score["score"]577 578 promoter_pct = promoter_hist[0]["promoter_pct"] if promoter_hist else None579 promoter_trend = None580 if len(promoter_hist) >= 2:581 latest_p, prior_p = promoter_hist[0]["promoter_pct"], promoter_hist[1]["promoter_pct"]582 if latest_p is not None and prior_p is not None:583 delta = latest_p - prior_p584 promoter_trend = "rising" if delta > 0.05 else "declining" if delta < -0.05 else "flat"585 586 fetched_at = cursor_by_symbol.get(sym)587 freshness_days = (now - fn._aware(fetched_at)).days if fetched_at else None588 589 full_cache[sym] = {590 "score": score["score"],591 "growth_score": score["growth_score"],592 "quality_score": score["quality_score"],593 "valuation_score": score["valuation_score"],594 "promoter_pct": promoter_pct,595 "promoter_trend": promoter_trend,596 "data_freshness_days": freshness_days,597 }598 finally:599 session.close()600 601 global _fundamentals_score_cache, _fundamentals_full_cache602 with _cache_lock:603 _fundamentals_score_cache = cache604 _fundamentals_full_cache = full_cache605 logger.info("[fundamentals] score cache refreshed: %d symbols", len(cache))606 except Exception as exc:607 logger.warning("[fundamentals] score cache refresh failed (non-fatal): %s", exc)608 609 610def _filter_by_fundamentals(items: list, min_fundamentals_score: Optional[float]) -> list:611 """Confluence filter (Phase 13d): keep only items whose bare symbol has a612 cached fundamentals score >= min_fundamentals_score. A no-op when the613 param is unset. Items without a cached score (not yet fetched) are614 excluded once a threshold is requested — they can't be confirmed to pass."""615 if min_fundamentals_score is None:616 return items617 out = []618 for item in items:619 bare = item.symbol.replace(".NS", "")620 score = _fundamentals_score_cache.get(bare)621 if score is not None and score >= min_fundamentals_score:622 out.append(item)623 return out624 625 626def _latest_setup_symbols() -> dict:627 """{bare_symbol: [screener names]} from the latest completed scan — lets the628 Smart Money view flag which deal stocks are also in a momentum/swing/SMC/629 reversal setup. Symbols are stored .NS-suffixed in the DB; we return bare."""630 out: dict[str, list[str]] = {}631 db = SessionLocal()632 try:633 run = _latest_completed_run(db)634 if not run:635 return out636 sources = [637 ("Momentum", db.query(MomentumResultORM.symbol).filter_by(run_id=run.id)),638 ("Swing", db.query(SwingSetupORM.symbol).filter_by(run_id=run.id)),639 ("SMC", db.query(SMCSetupORM.symbol).filter_by(run_id=run.id)),640 ("Reversal", db.query(ReversalWatchlistORM.symbol).filter_by(run_id=run.id)),641 ]642 for label, q in sources:643 for (sym,) in q.all():644 bare = (sym or "").replace(".NS", "").upper()645 if bare:646 out.setdefault(bare, []).append(label)647 except Exception as exc:648 logger.warning("[smart-money] setup cross-ref failed: %s", exc)649 finally:650 db.close()651 return out652 653 654def _refresh_setup_symbols() -> None:655 """Publish the latest scan's setup-symbol map into memory so the656 /api/smart-money endpoint can annotate deals without a live DB query."""657 mapping = _latest_setup_symbols()658 if mapping:659 global _setup_symbols660 with _cache_lock:661 _setup_symbols = mapping662 663 664def _refresh_universe_and_classification() -> None:665 """Daily cron target: rebuild the universe, refresh the classification, then666 pull the Smart Money snapshot — so new listings / index rebalances / deals667 are reflected across the Browse, Indices and Smart Money views."""668 try:669 n = refresh_universe()670 if n:671 _cache_clear(_stocks_list_cache)672 except Exception as exc:673 logger.warning("[cron] universe refresh failed: %s", exc)674 _build_classification()675 _build_upstox_instruments()676 _build_upstox_index_instruments()677 _refresh_smart_money()678 _refresh_macro_indicators()679 680 681# Scan tables are never wiped by app code — only an ephemeral SQLite restart682# used to do that for free. On persistent storage (Neon), every 30-min scan683# adds ~run's worth of rows across 4 tables forever, so retention is required684# to stay inside a free-tier storage cap. Keep enough runs to cover the news685# feed / breadth history's lookback with margin; irrelevant on SQLite dev686# (gets wiped on restart anyway) but harmless there too.687_RETENTION_RUNS = 200 # ~4 days at 30-min cadence688 689 690def _prune_old_scan_runs(db: Session) -> None:691 old_run_ids = [692 row[0] for row in db.query(ScanRun.id)693 .order_by(ScanRun.run_at.desc())694 .offset(_RETENTION_RUNS)695 .all()696 ]697 if not old_run_ids:698 return699 for model in (MomentumResultORM, SwingSetupORM, SMCSetupORM, ReversalWatchlistORM, VerdictCase):700 db.query(model).filter(model.run_id.in_(old_run_ids)).delete(synchronize_session=False)701 db.query(ScanRun).filter(ScanRun.id.in_(old_run_ids)).delete(synchronize_session=False)702 db.commit()703 logger.info("[scan] pruned %d old scan run(s)", len(old_run_ids))704 705 706def _execute_full_scan() -> Optional[int]:707 """708 Run all four scans and persist results to SQLite.709 Returns the run_id on success, None on failure.710 Thread-safe: a Lock prevents concurrent executions.711 """712 global _is_scanning713 714 if not _scan_lock.acquire(blocking=False):715 logger.info("[scheduler] Scan already running — skipped.")716 return None717 718 _is_scanning = True719 db: Session = SessionLocal()720 started_at = datetime.now(timezone.utc)721 t0 = time.perf_counter()722 723 scan_run = ScanRun(724 run_at=started_at,725 universe_size=len(NIFTY_500_UNIVERSE),726 status="running",727 )728 db.add(scan_run)729 db.commit()730 db.refresh(scan_run)731 run_id = scan_run.id732 logger.info("[scan] Starting full scan run_id=%d on %d stocks", run_id, len(NIFTY_500_UNIVERSE))733 734 try:735 # Fetch 18-month OHLCV for the whole universe ONCE and share it across736 # momentum, the SMC loop, and the reversal scan — avoids downloading the737 # universe 2-3x per scan (was: batch in momentum + per-symbol in SMC +738 # batch again in reversal).739 universe_dfs = _fetch_ohlcv_batch(NIFTY_500_UNIVERSE, period="18mo")740 741 # ── 1. Momentum ──────────────────────────────────────────────742 momentum_results, rs_by_symbol, all_momentum_data = scan_momentum(NIFTY_500_UNIVERSE, dfs=universe_dfs)743 744 # Per-stock snapshot for the Browse page — every scanned stock's latest745 # close + 1D change, keyed by bare symbol. Free: derived from data we746 # already computed. Published to the in-memory cache for /api/browse.747 snapshot = {}748 for md in all_momentum_data:749 try:750 snapshot[md.symbol.replace(".NS", "")] = {751 "change_pct": _f(md.change_pct),752 "close": _f(md.close),753 }754 except Exception:755 continue756 if snapshot:757 global _universe_snapshot758 with _cache_lock:759 _universe_snapshot = snapshot760 761 # Per-index equal-weight constituent proxies for the Indices Heatmap —762 # free, derived from the same price window + the index-membership map.763 try:764 import indices_nse765 membership = indices_nse.load_classification().get("membership", {})766 if membership:767 proxies = _compute_index_proxies(universe_dfs, membership)768 global _index_proxy769 with _cache_lock:770 _index_proxy = proxies771 except Exception as exc:772 logger.warning("[index-proxy] compute failed (non-fatal): %s", exc)773 for md in momentum_results:774 db.add(MomentumResultORM(775 run_id=run_id,776 symbol=md.symbol,777 close_price=_f(md.close),778 prev_close=_f(md.prev_close),779 change_pct=_f(md.change_pct),780 sma_50=_f(md.sma_50),781 sma_150=_f(md.sma_150),782 sma_200=_f(md.sma_200),783 sma_200_rising=bool(md.sma_200_rising),784 trend_aligned=bool(md.trend_aligned),785 rsi_14=_f(md.rsi_14),786 rs_score_raw=_f(md.rs_score_raw),787 rs_rating=_f(md.rs_rating),788 atr_14=_f(md.atr_14),789 high_52w=_f(md.high_52w),790 low_52w=_f(md.low_52w),791 pct_from_high=_f(md.pct_from_52w_high),792 pct_from_low=_f(md.pct_above_52w_low),793 volume=_f(md.volume),794 vol_20d_avg=_f(md.vol_20d_avg),795 volume_ratio=_f(md.volume_ratio),796 within_25pct_high=bool(md.within_25pct_of_high),797 above_30pct_low=bool(md.above_30pct_of_low),798 vcp_score=_f(md.vcp_score),799 vcp_contractions=int(md.vcp_contractions),800 vcp_stage=md.vcp_stage or None,801 vcp_pct_from_pivot=_f(md.vcp_pct_from_pivot) if md.vcp_pct_from_pivot is not None else None,802 ))803 804 # ── 2. Swing setups — independent of Minervini, runs on the full universe.805 # ATR-based entry/SL/target is valid for any stock; it should not require806 # passing the trend template. Overlap with momentum is a bonus, not a gate.807 swing_setups = scan_swing_setups(all_momentum_data, dfs=universe_dfs)808 for s in swing_setups:809 db.add(SwingSetupORM(810 run_id=run_id,811 symbol=s.symbol,812 entry_price=_f(s.entry_price),813 stop_loss=_f(s.stop_loss),814 target_1=_f(s.target_1),815 target_2=_f(s.target_2),816 atr_14=_f(s.atr_14),817 risk_pct=_f(s.risk_pct),818 rr_ratio=_f(s.rr_ratio),819 position_size_pct=_f(s.position_size_pct),820 rsi_14=_f(s.rsi_14),821 rs_rating=_f(s.rs_rating),822 volume_ratio=_f(s.volume_ratio),823 bars_since_signal=s.bars_since_signal,824 is_stale=s.is_stale,825 ))826 827 # ── 3. SMC setups — independent of Minervini, runs on the full universe.828 # analyse_stock is stateless (pure in-memory pandas/numpy per symbol),829 # so the 504 calls are parallelised with a thread pool. numpy releases830 # the GIL for many operations, giving real concurrency on multi-core.831 # DB writes happen serially after the parallel batch (SQLAlchemy session832 # is not thread-safe).833 smc_results = []834 def _smc_worker(item):835 sym, df = item836 return analyse_stock(sym, df, rs_rating=rs_by_symbol.get(sym))837 838 with ThreadPoolExecutor(max_workers=8) as pool:839 futures = {pool.submit(_smc_worker, item): item[0]840 for item in universe_dfs.items()}841 for fut in as_completed(futures):842 r = fut.result()843 if not r.error:844 smc_results.append(r)845 846 smc_count = 0847 for result in smc_results:848 smc_count += 1849 ev = result.last_structure850 ob = result.nearest_bull_ob851 fvg = result.nearest_bull_fvg852 lp = result.nearest_liquidity853 pdz = result.pd_zone854 db.add(SMCSetupORM(855 run_id=run_id,856 symbol=result.symbol,857 confluence_score=result.confluence_score,858 last_structure_type=ev.event_type if ev else None,859 structure_direction=ev.direction if ev else None,860 structure_price_level=_f(ev.price_level) if ev else None,861 structure_bar_date=ev.timestamp if ev else None,862 ob_active=ob is not None,863 ob_type=ob.bias if ob else None,864 ob_top=_f(ob.top) if ob else None,865 ob_bottom=_f(ob.bottom) if ob else None,866 ob_is_breaker=bool(ob.is_breaker) if ob else False,867 ob_formed_at=ob.formed_at if ob else None,868 fvg_active=fvg is not None,869 fvg_type=fvg.fvg_type if fvg else None,870 fvg_top=_f(fvg.top) if fvg else None,871 fvg_bottom=_f(fvg.bottom) if fvg else None,872 fvg_midpoint=_f(fvg.midpoint) if fvg else None,873 fvg_formed_at=fvg.formed_at if fvg else None,874 pd_zone=pdz.current_zone if pdz else None,875 swing_high=_f(pdz.swing_high) if pdz else None,876 swing_low=_f(pdz.swing_low) if pdz else None,877 nearest_liq_type=lp.liq_type if lp else None,878 nearest_liq_price=_f(lp.price) if lp else None,879 liq_swept=bool(lp.swept) if lp else False,880 all_order_blocks=[881 {"bias": o.bias, "top": o.top, "bottom": o.bottom,882 "active": o.active, "is_breaker": o.is_breaker}883 for o in result.order_blocks884 ],885 all_fvgs=[886 {"type": f.fvg_type, "top": f.top, "bottom": f.bottom, "active": f.active}887 for f in result.fvgs888 ],889 all_structure=[890 {"type": e.event_type, "direction": e.direction,891 "price": e.price_level, "date": e.timestamp.isoformat()}892 for e in result.structure_events893 ],894 all_liquidity=[895 {"type": p.liq_type, "price": p.price, "swept": p.swept}896 for p in result.liquidity_pools897 ],898 score_breakdown=result.score_breakdown,899 ))900 901 # ── 3b. Composite verdict + Bull/Bear case (Feature 1+2) — rule-based,902 # no LLM. Runs for the whole universe since it's cheap pure903 # computation over data already in memory from steps 1-3.904 # Then: auto paper-trading open/close off this scan's swing905 # signals (Feature 3), and decision-log writes for verdict906 # changes + paper-trade events (Feature 5). All non-fatal —907 # a failure here must never take the core scan down with it.908 try:909 swing_by_symbol = {s.symbol: s for s in swing_setups}910 smc_by_symbol = {r.symbol: r for r in smc_results}911 912 # Bulk-fetch recent material news once (not per-symbol) — same913 # "compute once, look up from memory" convention as914 # _fundamentals_score_cache below.915 _news_cutoff = datetime.now(timezone.utc) - timedelta(days=5)916 news_by_symbol: dict[str, list] = {}917 for item in (918 db.query(NewsItemORM)919 .filter(NewsItemORM.materiality == "material", NewsItemORM.ingested_at >= _news_cutoff)920 .all()921 ):922 if item.symbol:923 news_by_symbol.setdefault(item.symbol, []).append(item)924 925 def _smc_adapter(result):926 if result is None:927 return None928 ev = result.last_structure929 lp = result.nearest_liquidity930 pdz = result.pd_zone931 return SimpleNamespace(932 last_structure=SimpleNamespace(type=ev.event_type, direction=ev.direction) if ev else None,933 pd_zone=(pdz.current_zone if pdz else None),934 nearest_bull_ob=result.nearest_bull_ob,935 nearest_bull_fvg=result.nearest_bull_fvg,936 nearest_liquidity=SimpleNamespace(type=lp.liq_type, swept=lp.swept) if lp else None,937 )938 939 prior_run = _latest_completed_run(db)940 prior_verdicts: dict[str, str] = {}941 prior_smc_scores: dict[str, float] = {}942 prior_passing_symbols: set = set()943 if prior_run:944 prior_verdicts = {945 v.symbol: v.verdict946 for v in db.query(VerdictCase.symbol, VerdictCase.verdict)947 .filter(VerdictCase.run_id == prior_run.id).all()948 }949 prior_smc_scores = {950 s.symbol: s.confluence_score951 for s in db.query(SMCSetupORM.symbol, SMCSetupORM.confluence_score)952 .filter(SMCSetupORM.run_id == prior_run.id).all()953 }954 prior_passing_symbols = {955 m.symbol for m in db.query(MomentumResultORM.symbol)956 .filter(MomentumResultORM.run_id == prior_run.id).all()957 }958 959 verdict_changes: list[tuple] = [] # (symbol, old, new, score)960 for md in all_momentum_data:961 bare = md.symbol.replace(".NS", "")962 signals = verdict.evaluate_signals(963 md,964 swing=swing_by_symbol.get(md.symbol),965 smc=_smc_adapter(smc_by_symbol.get(md.symbol)),966 fundamentals=_fundamentals_full_cache.get(bare),967 recent_news=news_by_symbol.get(bare) or news_by_symbol.get(md.symbol),968 )969 case = verdict.build_verdict_and_case(signals)970 db.add(VerdictCase(971 run_id=run_id, symbol=md.symbol,972 verdict=case["verdict"], score=case["score"],973 bull_points=case["bull_points"], bear_points=case["bear_points"],974 bull_confidence=case["bull_confidence"], bear_confidence=case["bear_confidence"],975 invalidation_bull=case["invalidation_bull"], invalidation_bear=case["invalidation_bear"],976 ))977 prior = prior_verdicts.get(md.symbol)978 if prior is not None and prior != case["verdict"]:979 verdict_changes.append((md.symbol, prior, case["verdict"], case["score"]))980 981 db.commit()982 logger.info("[verdict] computed %d verdicts, %d changed since last run",983 len(all_momentum_data), len(verdict_changes))984 except Exception as exc:985 logger.warning("[verdict] computation failed (non-fatal): %s", exc)986 db.rollback()987 verdict_changes = []988 989 # Each strategy's open/close call gets its OWN try/except below —990 # deliberately, after a real incident: an AttributeError in the991 # momentum path (fixed separately) was silently swallowed by a992 # single shared try/except that wrapped swing+smc+momentum993 # together, and the swallowing masked the bug for an entire scan994 # cycle. One strategy failing must not blind the others.995 try:996 auth_db = auth.AuthSession()997 try:998 for symbol, old, new, score in verdict_changes:999 decision_log.log_entry(1000 auth_db, symbol, "verdict_change",1001 f"{symbol}: {old} -> {new} (score {score:.0f})",1002 detail={"old": old, "new": new, "score": score},1003 )1004 1005 opened = 01006 1007 try:1008 opened += paper_trading.open_new_positions(auth_db, swing_setups, strategy="swing")1009 except Exception as exc:1010 auth_db.rollback() # keep the shared session usable for what follows1011 logger.warning("[paper-trading:swing] open failed (non-fatal): %s", exc)1012 1013 try:1014 # SMC: synthetic "setup" rows for symbols newly crossing1015 # into A+ (score>=75, wasn't already >=75 last run) —1016 # reuses the same 3xATR-stop / 1.5R-2.5R-target bracket1017 # convention scan_swing_setups already uses, since1018 # there's no separate SMC backtest to source levels from.1019 momentum_by_symbol = {m.symbol: m for m in all_momentum_data}1020 smc_candidates = []1021 for sym, result in smc_by_symbol.items():1022 if result.confluence_score < paper_trading.SMC_ENTRY_GRADE_MIN:1023 continue1024 if prior_smc_scores.get(sym, 0) >= paper_trading.SMC_ENTRY_GRADE_MIN:1025 continue1026 md = momentum_by_symbol.get(sym)1027 if md is None or not md.atr_14 or md.atr_14 <= 0:1028 continue1029 entry = md.close1030 stop = entry - 3.0 * md.atr_141031 risk = entry - stop1032 if risk <= 0:1033 continue1034 smc_candidates.append(SimpleNamespace(1035 symbol=sym, bars_since_signal=0,1036 entry_price=entry, stop_loss=stop,1037 target_1=entry + 1.5 * risk, target_2=entry + 2.5 * risk,1038 ))1039 opened += paper_trading.open_new_positions(auth_db, smc_candidates, strategy="smc")1040 except Exception as exc:1041 auth_db.rollback()1042 logger.warning("[paper-trading:smc] open failed (non-fatal): %s", exc)1043 1044 try:1045 # Momentum: membership-based, opens on new template1046 # entrants, closes when a held symbol drops out.1047 price_by_symbol = {m.symbol: m.close for m in all_momentum_data}1048 current_passing_symbols = {r.symbol for r in momentum_results}1049 opened += paper_trading.open_momentum_positions(1050 auth_db, momentum_results, prior_passing_symbols)1051 paper_trading.close_momentum_positions(1052 auth_db, current_passing_symbols, price_by_symbol)1053 except Exception as exc:1054 auth_db.rollback()1055 logger.warning("[paper-trading:momentum] open/close failed (non-fatal): %s", exc)1056 1057 closed_before = auth_db.query(paper_trading.PaperPosition).filter_by(status="closed").count()1058 try:1059 paper_trading.check_and_close_positions(auth_db, universe_dfs)1060 except Exception as exc:1061 auth_db.rollback()1062 logger.warning("[paper-trading:bracket] close failed (non-fatal): %s", exc)1063 1064 if opened:1065 decision_log.log_entry(1066 auth_db, "PORTFOLIO", "paper_open",1067 f"{opened} new paper position(s) opened this scan",1068 )1069 closed_after = auth_db.query(paper_trading.PaperPosition).filter_by(status="closed").count()1070 if closed_after > closed_before:1071 decision_log.log_entry(1072 auth_db, "PORTFOLIO", "paper_close",1073 f"{closed_after - closed_before} paper position(s) closed this scan",1074 )1075 finally:1076 auth_db.close()1077 except Exception as exc:1078 logger.warning("[paper-trading] scan hook failed (non-fatal): %s", exc)1079 1080 # ── 4. Reversal watchlist ─────────────────────────────────────1081 reversal_candidates = scan_reversal_watchlist(NIFTY_500_UNIVERSE, dfs=universe_dfs)1082 for rc in reversal_candidates:1083 db.add(ReversalWatchlistORM(1084 run_id=run_id,1085 symbol=rc.symbol,1086 cross_date=rc.cross_date,1087 days_above_200=int(rc.days_above_200),1088 sma_200=_f(rc.sma_200),1089 rsi_14=_f(rc.rsi_14),1090 rsi_improving=bool(rc.rsi_improving),1091 volume_on_cross=_f(rc.volume_on_cross),1092 close_price=_f(rc.close),1093 rs_rating=_f(rc.rs_rating),1094 ))1095 1096 # ── 4b. Reversal paper trading — fixed-horizon, needs reversal_candidates1097 # which only exists after step 4 runs (non-fatal, same as 3b).1098 try:1099 auth_db = auth.AuthSession()1100 try:1101 try:1102 paper_trading.open_reversal_positions(auth_db, reversal_candidates)1103 except Exception as exc:1104 auth_db.rollback()1105 logger.warning("[paper-trading:reversal] open failed (non-fatal): %s", exc)1106 try:1107 paper_trading.close_reversal_positions(auth_db, universe_dfs)1108 except Exception as exc:1109 auth_db.rollback()1110 logger.warning("[paper-trading:reversal] close failed (non-fatal): %s", exc)1111 finally:1112 auth_db.close()1113 except Exception as exc:1114 logger.warning("[paper-trading:reversal] scan hook failed (non-fatal): %s", exc)1115 1116 # ── 5. Market breadth — reconstruct the historical curve from the same1117 # OHLCV batch (cheap: vectorised pandas over data already in memory).1118 try:1119 breadth = compute_breadth_history(universe_dfs, lookback_days=180)1120 global _breadth_history, _breadth_meta1121 with _cache_lock:1122 _breadth_history = breadth1123 _breadth_meta = {1124 "computed_at": datetime.now(timezone.utc).isoformat(),1125 "run_id": run_id,1126 "points": len(breadth),1127 }1128 logger.info("[scan] breadth history computed — %d trading days", len(breadth))1129 except Exception as exc:1130 logger.warning("[scan] breadth computation failed (non-fatal): %s", exc)1131 1132 # ── 6. Walk-forward backtest — at most once per trading day, in its1133 # own thread (it fetches a longer 3-year window for itself).1134 _maybe_schedule_backtest(universe_dfs)1135 1136 duration = round(time.perf_counter() - t0, 2)1137 scan_run.status = "completed"1138 scan_run.duration_sec = duration1139 db.commit()1140 1141 try:1142 _prune_old_scan_runs(db)1143 except Exception as exc:1144 logger.warning("[scan] retention prune failed (non-fatal): %s", exc)1145 1146 # Publish this run's setup-symbol map for Smart Money cross-referencing —1147 # the endpoint reads this in-memory dict instead of querying the DB live.1148 try:1149 _refresh_setup_symbols()1150 except Exception as exc:1151 logger.warning("[scan] setup-symbol publish failed (non-fatal): %s", exc)1152 1153 logger.info(1154 "[scan] run_id=%d completed in %.1fs — momentum=%d swing=%d smc=%d reversal=%d",1155 run_id, duration, len(momentum_results), len(swing_setups),1156 smc_count, len(reversal_candidates),1157 )1158 return run_id1159 1160 except Exception as exc:1161 logger.error("[scan] run_id=%d failed: %s", run_id, exc, exc_info=True)1162 try:1163 scan_run.status = "failed"1164 scan_run.error_msg = str(exc)1165 db.commit()1166 except Exception:1167 db.rollback()1168 return None1169 1170 finally:1171 db.close()1172 _is_scanning = False1173 _scan_lock.release()1174 1175 1176# ============================================================1177# LIFESPAN — scheduler + initial scan on startup1178# ============================================================1179 1180_scheduler = BackgroundScheduler(daemon=True)1181 1182@asynccontextmanager1183async def lifespan(app: FastAPI):1184 init_db(engine)1185 auth.init_auth_db()1186 import fundamentals_nse1187 fundamentals_nse.init_fundamentals_db()1188 paper_trading.init_paper_trading_db()1189 decision_log.init_decision_log_db()1190 logger.info("Database tables created / verified.")1191 1192 # Build the full NSE universe (nsepython + liquidity filter), then scan —1193 # all off the boot path so the healthcheck passes immediately.1194 threading.Thread(target=_startup_build_and_scan, daemon=True, name="startup-scan").start()1195 1196 # News Monitor gets its own thread — independent of the universe/scan1197 # thread above, which can take ~10-20 min on a cold boot. Without this,1198 # every redeploy (backend/ push -> HF Space restart, ephemeral SQLite1199 # wiped) would leave the news feed empty for as long as the scan runs,1200 # defeating the "beat TV news" point of the fast-poll design.