SafeVixAI/SafeVixAI-Dataset-Hub

1#!/usr/bin/env python32"""3SafeVixAI — Living Master Document Auto-Updater4================================================5Triggered on push to docs/root .md files via GitHub Actions.6Fetches live data from GitHub API, Render, and Vercel,7then rewrites PART C of docs/SafeVixAI_MASTER.docx.8 9Usage:10    python scripts/update_master_doc.py11 12Environment:13    GITHUB_TOKEN  — GitHub Personal Access Token (auto-provided by Actions)14"""15 16import os17import sys18import json19import requests20from datetime import datetime, timezone, timedelta21from docx import Document22from docx.shared import Pt, RGBColor, Inches23from docx.enum.text import WD_ALIGN_PARAGRAPH24from docx.enum.table import WD_TABLE_ALIGNMENT25 26# Inject project root to sys.path to access alert_service singleton27sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))28try:29    from alert_service import get_alert_service30except Exception:31    get_alert_service = None32 33# Fix Windows cp1252 encoding crashes with emoji print statements34try:35    sys.stdout.reconfigure(encoding='utf-8', errors='replace')36except Exception:37    pass38 39# ─── Configuration ──────────────────────────────────────────────────────────40 41GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN", "")42REPO = os.environ.get("GITHUB_REPOSITORY", "SafeVixAI/SafeVixAI")43GH_HEADERS = {44    "Accept": "application/vnd.github.v3+json",45}46if GITHUB_TOKEN:47    GH_HEADERS["Authorization"] = f"Bearer {GITHUB_TOKEN}"48IST = timezone(timedelta(hours=5, minutes=30))49 50SERVICES = {51    "Backend API": os.environ.get("BACKEND_HEALTH_URL", "https://safevixai-api.onrender.com/health"),52    "Chatbot Service": os.environ.get(53        "CHATBOT_HEALTH_URL",54        "https://safevixai-chatbot-service.onrender.com/health",55    ),56    "Frontend (Vercel)": os.environ.get("FRONTEND_URL", "https://safevixai.vercel.app"),57}58 59MASTER_DOC_PATH = os.path.join(60    os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))),61    "docs",62    "SafeVixAI_MASTER.docx",63)64 65# ─── Data Fetchers ──────────────────────────────────────────────────────────66 67 68def fetch_open_issues():69    """Fetch all open GitHub issues (up to 50)."""70    try:71        r = requests.get(72            f"https://api.github.com/repos/{REPO}/issues",73            params={"state": "open", "per_page": 50},74            headers=GH_HEADERS,75            timeout=15,76        )77        if r.ok:78            # Filter out pull requests (they also show up as issues)79            return [i for i in r.json() if "pull_request" not in i]80        print(f"  ⚠ GitHub Issues API returned {r.status_code}")81        return []82    except Exception as e:83        print(f"  ⚠ GitHub Issues fetch failed: {e}")84        return []85 86 87def fetch_recent_commits(n=10):88    """Fetch the last N commits from the default branch."""89    try:90        r = requests.get(91            f"https://api.github.com/repos/{REPO}/commits",92            params={"per_page": n},93            headers=GH_HEADERS,94            timeout=15,95        )96        if not r.ok:97            print(f"  ⚠ GitHub Commits API returned {r.status_code}")98            return []99        return [100            {101                "sha": c["sha"][:7],102                "msg": c["commit"]["message"].split("\n")[0][:80],103                "author": c["commit"]["author"]["name"],104                "date": c["commit"]["author"]["date"][:10],105            }106            for c in r.json()107        ]108    except Exception as e:109        print(f"  ⚠ GitHub Commits fetch failed: {e}")110        return []111 112 113def fetch_service_health():114    """Ping each deployed service and record latency + status."""115    results = {}116    for name, url in SERVICES.items():117        try:118            r = requests.get(url, timeout=15)119            ms = int(r.elapsed.total_seconds() * 1000)120            # Try to extract version from JSON health response121            version = "?"122            content_type = r.headers.get("content-type", "")123            if "json" in content_type:124                try:125                    version = r.json().get("version", "?")126                except Exception:127                    pass128            results[name] = {129                "status": "UP",130                "ms": ms,131                "version": version,132                "code": r.status_code,133            }134        except requests.exceptions.Timeout:135            results[name] = {"status": "DOWN", "error": "Connection timed out (15s)"}136        except requests.exceptions.ConnectionError:137            results[name] = {"status": "DOWN", "error": "Connection refused / DNS failure"}138        except Exception as e:139            results[name] = {"status": "DOWN", "error": str(e)[:60]}140 141        if results[name]["status"] == "DOWN" and get_alert_service:142            try:143                get_alert_service().alert_external_api_failed(144                    service_name=f"Deployment ping ({name})",145                    endpoint=url,146                    status_code=0,147                    error_msg=results[name]["error"],148                )149            except Exception:150                pass151    return results152 153 154def fetch_workflow_runs():155    """Fetch the last 5 GitHub Actions workflow runs."""156    try:157        r = requests.get(158            f"https://api.github.com/repos/{REPO}/actions/runs",159            params={"per_page": 8},160            headers=GH_HEADERS,161            timeout=15,162        )163        if not r.ok:164            print(f"  ⚠ GitHub Actions API returned {r.status_code}")165            return []166        return [167            {168                "name": run["name"],169                "status": run["status"],170                "conclusion": run.get("conclusion", "in_progress"),171                "branch": run["head_branch"],172                "date": run["created_at"][:10],173            }174            for run in r.json().get("workflow_runs", [])[:8]175        ]176    except Exception as e:177        print(f"  ⚠ GitHub Actions fetch failed: {e}")178        return []179 180 181def fetch_repo_stats():182    """Fetch repository-level statistics (stars, forks, size)."""183    try:184        r = requests.get(185            f"https://api.github.com/repos/{REPO}",186            headers=GH_HEADERS,187            timeout=15,188        )189        if r.ok:190            data = r.json()191            return {192                "stars": data.get("stargazers_count", 0),193                "forks": data.get("forks_count", 0),194                "open_issues": data.get("open_issues_count", 0),195                "size_kb": data.get("size", 0),196                "default_branch": data.get("default_branch", "main"),197                "updated_at": data.get("updated_at", "")[:10],198            }199        return {}200    except Exception:201        return {}202 203 204# ─── DOCX Helpers ───────────────────────────────────────────────────────────205 206 207def add_colored_heading(doc, text, level, hex_color="1A5C38"):208    """Add a heading with a custom color."""209    para = doc.add_heading(text, level=level)210    for run in para.runs:211        run.font.color.rgb = RGBColor(212            int(hex_color[0:2], 16),213            int(hex_color[2:4], 16),214            int(hex_color[4:6], 16),215        )216    return para217 218 219def add_styled_para(doc, text, bold=False, italic=False, font_size=10):220    """Add a paragraph with optional styling."""221    para = doc.add_paragraph()222    run = para.add_run(text)223    run.bold = bold224    run.italic = italic225    run.font.size = Pt(font_size)226    return para227 228 229def add_status_table(doc, headers, rows):230    """Add a formatted table to the document."""231    table = doc.add_table(rows=1 + len(rows), cols=len(headers))232    try:233        table.style = "Light Grid Accent 1"234    except KeyError:235        table.style = "Table Grid"236    table.alignment = WD_TABLE_ALIGNMENT.CENTER237 238    # Header row239    for j, header in enumerate(headers):240        cell = table.rows[0].cells[j]241        cell.text = header242        for para in cell.paragraphs:243            for run in para.runs:244                run.bold = True245                run.font.size = Pt(9)246 247    # Data rows248    for i, row in enumerate(rows):249        for j, val in enumerate(row):250            cell = table.rows[i + 1].cells[j]251            cell.text = str(val)252            for para in cell.paragraphs:253                for run in para.runs:254                    run.font.size = Pt(9)255 256    return table257 258 259# ─── Main Update Logic ─────────────────────────────────────────────────────260 261 262def update_part_c(doc_path: str):263    """264    Open the master DOCX, find the PART C marker,265    delete everything after it, and write fresh live data.266    """267    if not os.path.exists(doc_path):268        print(f"✘ Master doc not found at: {doc_path}")269        sys.exit(1)270 271    doc = Document(doc_path)272    now = datetime.now(IST).strftime("%Y-%m-%d %H:%M IST")273 274    # ── Find PART C marker and clear everything after it ────────────────275    part_c_idx = None276    # Iterate backwards so we hit the actual heading instead of the Table of Contents277    for i in range(len(doc.paragraphs) - 1, -1, -1):278        para = doc.paragraphs[i]279        if "PART C" in para.text and "LIVE" in para.text.upper() and para.style.name.startswith("Heading"):280            part_c_idx = i281            break282 283    if part_c_idx is None:284        print("⚠ PART C marker not found — appending at end")285    else:286        # Remove all elements after PART C heading, but preserve sectPr287        body = doc.element.body288        part_c_element = doc.paragraphs[part_c_idx]._element289        ns = "{http://schemas.openxmlformats.org/wordprocessingml/2006/main}"290        # Find all siblings after the PART C paragraph291        found = False292        elements_to_remove = []293        for child in body:294            if child is part_c_element:295                found = True296                continue297            if found:298                # Never remove sectPr — it holds page layout metadata299                if child.tag == f"{ns}sectPr":300                    continue301                elements_to_remove.append(child)302        for elem in elements_to_remove:303            body.remove(elem)304 305    print(f"📡 Fetching live data at {now}...")306 307    # ── WRITE FRESH PART C CONTENT ──────────────────────────────────────308 309    # Timestamp310    add_styled_para(doc, f"🕐 Last auto-updated: {now}", bold=True, font_size=11)311    add_styled_para(312        doc,313        "This section is automatically rewritten on push to docs/ or root .md files via GitHub Actions. "314        "Do not edit manually — changes will be overwritten.",315        italic=True,316        font_size=9,317    )318 319    # ── Section 1: Repository Overview ──────────────────────────────────320    add_colored_heading(doc, "Repository Overview", 2)321    stats = fetch_repo_stats()322    if stats:323        add_status_table(324            doc,325            ["Metric", "Value"],326            [327                ["⭐ Stars", stats.get("stars", "N/A")],328                ["🍴 Forks", stats.get("forks", "N/A")],329                ["📦 Repo Size", f"{stats.get('size_kb', 0) // 1024} MB"],330                ["🔀 Default Branch", stats.get("default_branch", "main")],331                ["📅 Last Updated", stats.get("updated_at", "N/A")],332                ["🐛 Open Issues", stats.get("open_issues", "N/A")],333            ],334        )335    else:336        add_styled_para(doc, "⚠ Could not fetch repository statistics.", italic=True)337 338    # ── Section 2: Service Health ───────────────────────────────────────339    add_colored_heading(doc, "Deployment & Service Health", 2)340    health = fetch_service_health()341    health_rows = []342    for name, h in health.items():343        if h["status"] == "UP":344            status = "✅ UP"345            detail = f"{h.get('ms', '?')}ms | HTTP {h.get('code', '?')} | v{h.get('version', '?')}"346        else:347            status = "🔴 DOWN"348            detail = h.get("error", "Unknown error")349        health_rows.append([name, status, detail])350    add_status_table(doc, ["Service", "Status", "Details"], health_rows)351 352    up_count = sum(1 for h in health.values() if h["status"] == "UP")353    total = len(health)354    add_styled_para(355        doc,356        f"Overall: {up_count}/{total} services operational.",357        bold=True,358        font_size=10,359    )360 361    # ── Section 3: CI/CD Pipeline Status ────────────────────────────────362    add_colored_heading(doc, "Recent CI/CD Runs", 2)363    runs = fetch_workflow_runs()364    if runs:365        ci_rows = []366        for run in runs:367            conclusion = run.get("conclusion") or run["status"]368            if conclusion == "success":369                icon = "✅"370            elif conclusion == "failure":371                icon = "🔴"372            elif conclusion == "cancelled":373                icon = "⚪"374            else:375                icon = "⏳"376            ci_rows.append(377                [378                    f"{icon} {run['name']}",379                    run["branch"],380                    conclusion,381                    run["date"],382                ]383            )384        add_status_table(doc, ["Workflow", "Branch", "Result", "Date"], ci_rows)385    else:386        add_styled_para(doc, "No recent CI/CD runs found.", italic=True)387 388    # ── Section 4: Open GitHub Issues ───────────────────────────────────389    add_colored_heading(doc, "Open GitHub Issues", 2)390    issues = fetch_open_issues()391    critical = [392        i393        for i in issues394        if any(395            l["name"].lower() in ["critical", "bug", "p0", "security"]396            for l in i.get("labels", [])397        )398    ]399    add_styled_para(400        doc,401        f"Total open: {len(issues)} | Critical/Bug: {len(critical)}",402        bold=True,403    )404 405    if issues:406        issue_rows = []407        for issue in issues[:25]:408            labels = ", ".join(l["name"] for l in issue.get("labels", []))409            assignee = (issue.get("assignee") or {}).get("login", "—")410            issue_rows.append(411                [412                    f"#{issue['number']}",413                    issue["title"][:60],414                    labels or "—",415                    assignee,416                ]417            )418        add_status_table(doc, ["#", "Title", "Labels", "Assignee"], issue_rows)419        if len(issues) > 25:420            add_styled_para(421                doc,422                f"... and {len(issues) - 25} more → github.com/{REPO}/issues",423                italic=True,424                font_size=9,425            )426    else:427        add_styled_para(doc, "🎉 No open issues — all clear!", font_size=10)428 429    # ── Section 5: Recent Commits ───────────────────────────────────────430    add_colored_heading(doc, "Recent Commits (last 10)", 2)431    commits = fetch_recent_commits(10)432    if commits:433        commit_rows = []434        for c in commits:435            commit_rows.append([c["sha"], c["date"], c["msg"], c["author"]])436        add_status_table(doc, ["SHA", "Date", "Message", "Author"], commit_rows)437    else:438        add_styled_para(doc, "No commits found.", italic=True)439 440    # ── Section 6: Feature Completion Matrix ────────────────────────────441    add_colored_heading(doc, "Feature Completion Status", 2)442    add_status_table(443        doc,444        ["Module", "Status", "Confidence"],445        [446            ["Emergency Locator (GPS + SOS)", "✅ Production", "95%"],447            ["AI Chatbot (11 LLM providers)", "✅ Production", "90%"],448            ["Challan Calculator (DuckDB)", "✅ Production", "95%"],449            ["Road Reporter (RoadWatch)", "✅ Production", "90%"],450            ["Crash Detection (DeviceMotion)", "✅ Production", "85%"],451            ["Offline AI (WebLLM Phi-3)", "✅ Production", "85%"],452            ["Live Family Tracking", "✅ Production", "80%"],453            ["Bystander Mode", "✅ Production", "80%"],454            ["PWA (Offline + Install)", "✅ Production", "90%"],455            ["Waze CIFS Feed", "✅ Production", "85%"],456        ],457    )458 459    # ── Section 7: Production Monitoring & Alerting ─────────────────────460    add_colored_heading(doc, "Production Monitoring & Alerting", 2)461    add_styled_para(462        doc,463        "SafeVixAI uses alert_service.py (project root) for production failure notifications. "464        "Email alerts are sent via Gmail SMTP when critical systems fail.",465        font_size=10,466    )467    add_status_table(468        doc,469        ["Service", "Monitored By", "Trigger"],470        [471            ["9 LLM Providers", "chatbot/providers/router.py", "All fallback providers fail"],472            ["Backend APIs (Overpass, Nominatim, OSRM)", "chatbot/tools/__init__.py", "HTTP 5xx or timeout"],473            ["PostgreSQL/PostGIS Database", "backend/main.py", "/health returns DB unavailable"],474            ["Unhandled Backend Errors", "backend/main.py", "Any unhandled 500 exception"],475            ["Wiki Doc Generation", "scripts/wiki_manager.py", "5+ consecutive LLM failures"],476        ],477    )478    add_styled_para(479        doc,480        "Each alert includes 3 diagnostic solutions + 5-min cooldown per alert type.",481        italic=True,482        font_size=9,483    )484 485    # ── Section 8: Wiki Documentation Stats ─────────────────────────────486    add_colored_heading(doc, "Auto-Generated Wiki Documentation", 2)487    wiki_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))), "docs", "wiki", "content")488    wiki_count = 0489    mermaid_count = 0490    if os.path.isdir(wiki_dir):491        for root_d, _, files in os.walk(wiki_dir):492            for f in files:493                if f.endswith(".md"):494                    wiki_count += 1495                    try:496                        text = open(os.path.join(root_d, f), encoding="utf-8").read()497                        mermaid_count += text.count("```mermaid")498                    except Exception:499                        pass500    add_status_table(501        doc,502        ["Metric", "Value"],503        [504            ["Wiki Pages", str(wiki_count)],505            ["Mermaid Diagrams", str(mermaid_count)],506            ["Generation Method", "LLM (OpenRouter → Mistral → Gemini)"],507            ["CI Trigger", "Push to backend/chatbot/frontend/docs"],508            ["Source of Truth", "Code → LLM → Wiki (never manually edit)"],509        ],510    )511 512    # ── Footer ──────────────────────────────────────────────────────────513    doc.add_page_break()514    add_styled_para(515        doc,516        f"— End of Auto-Generated Section —\n"517        f"Generated by scripts/update_master_doc.py\n"518        f"Timestamp: {now}\n"519        f"Repository: github.com/{REPO}",520        italic=True,521        font_size=8,522    )523 524    # ── Save ────────────────────────────────────────────────────────────525    doc.save(doc_path)526    print(f"✅ Master doc updated at {now}")527    print(f"   Services: {up_count}/{total} UP")528    print(f"   Open issues: {len(issues)}")529    print(f"   Recent commits: {len(commits)}")530    print(f"   CI runs: {len(runs)}")531 532 533# ─── Entry Point ────────────────────────────────────────────────────────────534 535if __name__ == "__main__":536    if not GITHUB_TOKEN:537        print("⚠ GITHUB_TOKEN not set — GitHub API calls will be rate-limited")538    update_part_c(MASTER_DOC_PATH)539