sridattapradeep/India_Equity_Scanner
0
1"""2macro_indicators.py — Live-ish macro reference panel: RBI policy rates, GDP3growth, and best-effort CPI/WPI.4 5Three different sources, three different honesty levels — each snapshot6field is tagged with where it came from so the frontend can label it7correctly rather than implying everything is an equally-authoritative feed:8 9 - RBI policy rates: scraped directly off rbi.org.in's own "Current Rates"10 table (Policy Repo Rate, SDF, MSF, Bank Rate, Fixed Reverse Repo, CRR,11 SLR). Genuinely live — RBI updates this table itself.12 - GDP growth: FRED (St Louis Fed), series INDGDPRQPSMEI — real, current,13 quarterly India GDP growth. Requires a free FRED_API_KEY (no card).14 - CPI / WPI: FRED's India series for these are effectively abandoned15 (checked 2026-07-18: monthly CPI index last observation 2025-03, WPI16 last touched 2023) — not usable as "live". Instead, best-effort17 extracted from the News Monitor's own macro news headlines (market_news18 NewsItem rows), regex-pulling the most recent "X.XX%" mentioned near a19 CPI/WPI/inflation keyword. Labeled "from news coverage", not a direct20 official reading — it's whatever number a news outlet most recently21 reported, which is usually the real print but isn't independently22 verified against MOSPI's own release.23 24Standalone module (imports models + requests only) mirroring nse_live.py's25own design. Every fetcher is resilient: any failure logs and returns26None/{} — never fatal.27"""28 29from __future__ import annotations30 31import json32import logging33import os34import re35from datetime import datetime, timezone36 37import requests38 39from models import NewsItem40 41logger = logging.getLogger(__name__)42 43_RBI_HOME = "https://www.rbi.org.in/"44_RBI_HEADERS = {45 "User-Agent": (46 "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "47 "(KHTML, like Gecko) Chrome/120.0 Safari/537.36"48 ),49}50_RBI_RATES_BLOCK_RE = re.compile(51 r"Policy Rates.*?</table>.*?Reserve\s* Ratios.*?<table.*?</table>", re.S,52)53_RBI_RATE_ROW_RE = re.compile(54 r"<th[^>]*>\s*([^<]+?)\s*</th>\s*<td[^>]*>\s*:\s*([^<]+?)\s*</td>",55)56_RBI_RATE_KEYS = {57 "Policy Repo Rate": "repo_rate",58 "Standing Deposit Facility Rate": "sdf_rate",59 "Marginal Standing Facility Rate": "msf_rate",60 "Bank Rate": "bank_rate",61 "Fixed Reverse Repo Rate": "reverse_repo_rate",62 "CRR": "crr",63 "SLR": "slr",64}65 66_FRED_BASE = "https://api.stlouisfed.org/fred/series/observations"67_FRED_GDP_SERIES = "INDGDPRQPSMEI" # India real GDP growth, quarterly (checked current 2026-07-18)68 69_CACHE_FILE = os.path.join(os.path.dirname(__file__), "macro_snapshot.json")70_CACHE: dict | None = None71 72 73def fetch_rbi_policy_rates() -> dict:74 """RBI's own "Current Rates" table off rbi.org.in — resilient: {} on any75 failure (network, layout change — regex simply matches nothing)."""76 try:77 r = requests.get(_RBI_HOME, headers=_RBI_HEADERS, timeout=15)78 r.raise_for_status()79 html = r.text80 except Exception as exc:81 logger.warning("[macro] RBI rates fetch failed: %s", exc)82 return {}83 m = _RBI_RATES_BLOCK_RE.search(html)84 if not m:85 logger.warning("[macro] RBI rates table not found (layout change?)")86 return {}87 out: dict[str, float] = {}88 for label, value in _RBI_RATE_ROW_RE.findall(m.group(0)):89 key = _RBI_RATE_KEYS.get(label.strip())90 if not key:91 continue92 try:93 out[key] = float(value.strip().rstrip("%"))94 except ValueError:95 continue96 return out97 98 99def fetch_gdp_growth() -> dict:100 """Latest India GDP growth reading from FRED. Resilient: {} if101 FRED_API_KEY is unset or the request fails."""102 api_key = os.getenv("FRED_API_KEY")103 if not api_key:104 return {}105 try:106 r = requests.get(_FRED_BASE, params={107 "series_id": _FRED_GDP_SERIES, "api_key": api_key,108 "file_type": "json", "sort_order": "desc", "limit": 1,109 }, timeout=15)110 r.raise_for_status()111 obs = r.json().get("observations", [])112 except Exception as exc:113 logger.warning("[macro] FRED GDP fetch failed: %s", exc)114 return {}115 if not obs or obs[0].get("value") in (None, "."):116 return {}117 try:118 return {"value_pct": round(float(obs[0]["value"]), 2), "as_of": obs[0]["date"]}119 except (ValueError, KeyError):120 return {}121 122 123# Matches "<label word(s)> ... 4.38%" or "4.38% ... <label>" within a short124# window, case-insensitive. Keeps it a plain number extraction — no attempt125# to disambiguate multiple percentages in one headline beyond taking the126# one nearest the keyword.127_PCT_RE = re.compile(r"(\d{1,2}\.\d{1,2})\s*%")128 129 130def _extract_pct_near(text: str, keyword_re: re.Pattern) -> float | None:131 m = keyword_re.search(text)132 if not m:133 return None134 window = text[max(0, m.start() - 60): m.end() + 60]135 pct = _PCT_RE.search(window)136 return float(pct.group(1)) if pct else None137 138 139_CPI_KEYWORD_RE = re.compile(r"\bCPI\b|consumer price index|retail inflation", re.IGNORECASE)140_WPI_KEYWORD_RE = re.compile(r"\bWPI\b|wholesale price index", re.IGNORECASE)141 142 143def extract_cpi_wpi_from_news(db) -> dict:144 """Best-effort CPI/WPI reading from the most recent market_news headlines145 that mention them — see module docstring for why this exists instead of146 an official feed. Returns {} entries for whichever indicator has no147 recent match. Resilient: any failure returns {}."""148 try:149 rows = (150 db.query(NewsItem)151 .filter(NewsItem.source == "market_news")152 .order_by(NewsItem.ingested_at.desc())153 .limit(200)154 .all()155 )156 except Exception as exc:157 logger.warning("[macro] CPI/WPI news query failed: %s", exc)158 return {}159 160 out: dict[str, dict] = {}161 for indicator, keyword_re in (("cpi", _CPI_KEYWORD_RE), ("wpi", _WPI_KEYWORD_RE)):162 if indicator in out:163 continue164 for row in rows:165 pct = _extract_pct_near(row.headline, keyword_re)166 if pct is not None:167 out[indicator] = {168 "value_pct": pct,169 "headline": row.headline,170 "url": row.url,171 "as_of": (row.published_at or row.ingested_at).isoformat(),172 }173 break174 return out175 176 177def build_macro_snapshot(db) -> dict:178 """Fetch every macro source, assemble + persist the snapshot. Caches in179 memory and writes macro_snapshot.json. Resilient: any single source180 failing leaves that field empty, never blocks the others."""181 global _CACHE182 snapshot = {183 "rbi_rates": fetch_rbi_policy_rates(),184 "gdp_growth": fetch_gdp_growth(),185 "news_derived": extract_cpi_wpi_from_news(db),186 "built_at": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),187 }188 logger.info(189 "[macro] snapshot built: rbi_rates=%d gdp=%s cpi=%s wpi=%s",190 len(snapshot["rbi_rates"]), bool(snapshot["gdp_growth"]),191 "cpi" in snapshot["news_derived"], "wpi" in snapshot["news_derived"],192 )193 _CACHE = snapshot194 try:195 with open(_CACHE_FILE, "w", encoding="utf-8") as fh:196 json.dump(snapshot, fh)197 except Exception as exc:198 logger.warning("[macro] could not persist macro_snapshot.json: %s", exc)199 return snapshot200 201 202def load_macro_snapshot() -> dict:203 """In-memory cache -> macro_snapshot.json -> empty skeleton. Never fetches."""204 global _CACHE205 if _CACHE is not None:206 return _CACHE207 try:208 if os.path.exists(_CACHE_FILE):209 with open(_CACHE_FILE, encoding="utf-8") as fh:210 _CACHE = json.load(fh)211 return _CACHE212 except Exception as exc:213 logger.warning("[macro] could not read macro_snapshot.json: %s", exc)214 return {"rbi_rates": {}, "gdp_growth": {}, "news_derived": {}, "built_at": None}215 216 217if __name__ == "__main__":218 logging.basicConfig(level=logging.INFO, format="%(message)s")219 from sqlalchemy import create_engine220 from sqlalchemy.orm import sessionmaker221 engine = create_engine("sqlite:///./scanner.db")222 Session = sessionmaker(bind=engine)223 snap = build_macro_snapshot(Session())224 print(json.dumps(snap, indent=2, default=str))225