SafeVixAI/SafeVixAI-Dataset-Hub

1#!/usr/bin/env python32"""3SafeVixAI Wiki Manager v3 — Multi-LLM Documentation Generator with Email Alerts4 5Uses OpenRouter/Mistral/Gemini fallback chain to generate wiki docs from source code.6Falls back to AST-based stubs if no API key is available.7Sends email alerts on persistent failures with 3 solution suggestions.8 9Modes:10  python scripts/wiki_manager.py check        # Report coverage + staleness11  python scripts/wiki_manager.py fix          # Fix stale refs12  python scripts/wiki_manager.py generate     # Generate docs for new code (LLM or AST)13  python scripts/wiki_manager.py full         # fix + generate (CI mode)14  python scripts/wiki_manager.py update       # Re-generate outdated docs15 16Env:17  OPENROUTER_API_KEY — OpenRouter (primary, uses Gemini Flash via proxy)18  MISTRAL_API_KEY    — Mistral (secondary fallback)19  GOOGLE_API_KEY     — Gemini Direct (tertiary, rate-limited)20  ALERT_EMAIL        — Gmail address for failure alerts (optional)21  ALERT_EMAIL_PASSWORD — Gmail App Password for SMTP (optional)22"""23 24import os25import re26import sys27import json28import ast29import time30import smtplib31import textwrap32from pathlib import Path33from datetime import datetime34from email.mime.text import MIMEText35from urllib.request import Request, urlopen36from urllib.error import URLError, HTTPError37 38# Inject project root to sys.path to access alert_service singleton39sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))40try:41    from alert_service import get_alert_service42except Exception:43    get_alert_service = None44 45# ── Paths ────────────────────────────────────────────────────────────────────46ROOT = Path(__file__).resolve().parent.parent.parent47WIKI_CONTENT = ROOT / "docs" / "wiki" / "content"48WIKI_META = ROOT / "docs" / "wiki" / "meta" / "repowiki-metadata.json"49DOCS_DIR = ROOT / "docs"50 51# ── Ground Truth ─────────────────────────────────────────────────────────────52GROUND_TRUTH = {53    "embedding_model": "LocalHashEmbeddingFunction (SHA-256, zero ML dependency)",54    "intent_count": 9,55    "tool_count": 13,56    "provider_count": 9,57    "providers": [58        "Cerebras", "Gemini", "GitHub Models", "Groq", "Mistral",59        "NVIDIA NIM", "OpenRouter", "Sarvam AI", "Together AI"60    ],61    "endpoint_count": 28,62    "component_count": 45,63    "page_count": 16,64}65 66# ── Code → Wiki Section Mapping ──────────────────────────────────────────────67MODULE_MAP = {68    "backend/api": "API Reference",69    "backend/models": "Database Schema",70    "backend/services": "Project Overview/Core Modules Overview",71    "chatbot_service/providers": "AI Chatbot Service",72    "chatbot_service/tools": "AI Chatbot Service",73    "chatbot_service/agents": "AI Chatbot Service",74    "chatbot_service/models": "AI Chatbot Service",75    "frontend/app": "Frontend Application",76    "frontend/components": "Frontend Application/Component Library",77    "frontend/hooks": "Frontend Application",78    "frontend/lib": "Frontend Application",79    "data": "Data Management",80}81 82CODE_EXTENSIONS = {".py", ".ts", ".tsx", ".js", ".jsx"}83SKIP = {"__pycache__", "node_modules", ".next", ".git", "venv",84        "__init__.py", "conftest.py", "test_", "_test.", ".test.",85        "index", "layout", "loading", "not-found", "error"}86 87# ── Stale Fixes ──────────────────────────────────────────────────────────────88STALE_FIXES = [89    (r'\b11 LLMs?\b', '9 LLMs', 'LLM count'),90    (r'\b11 LLM providers?\b', '9 LLM providers', 'LLM provider count'),91    (r'11-provider', '9-provider', 'provider count'),92    (r'sentence-transformers/all-MiniLM-L6-v2', 'LocalHashEmbeddingFunction (zero-dependency)', 'embedding'),93    (r'sentence[- ]transformers', 'hash-based embeddings', 'embedding'),94    (r'SentenceTransformer\b', 'LocalHashEmbeddingFunction', 'embedding class'),95    (r'all-MiniLM-L6-v2', 'LocalHashEmbeddingFunction', 'embedding model'),96    (r'admin123', 'environment-sourced credentials', 'demo cred'),97    (r'mock-jwt-token-for-hackathon', 'environment-sourced JWT', 'demo token'),98    (r'mock-jwt', 'environment-sourced JWT', 'demo token'),99]100 101SAFE_LINE_PATTERNS = [102    r'@huggingface/transformers', r'huggingface\.co/datasets/SafeVixAI',103    r'SafeVixAI-Dataset-Hub', r'HF Inference API', r'HF_TOKEN', r'via HF_TOKEN',104]105 106 107def is_safe_line(line):108    return any(re.search(p, line, re.IGNORECASE) for p in SAFE_LINE_PATTERNS)109 110 111# ══════════════════════════════════════════════════════════════════════════════112#  EMAIL ALERT SYSTEM113# ══════════════════════════════════════════════════════════════════════════════114 115def send_alert(subject, details, context=""):116    """Send email alert on failure with 3 solution suggestions.117 118    Requires ALERT_EMAIL + ALERT_EMAIL_PASSWORD env vars (Gmail App Password).119    Falls back to console output if email isn't configured.120    """121    smtp_user = os.environ.get("ALERT_EMAIL", "")122    smtp_pass = os.environ.get("ALERT_EMAIL_PASSWORD", "")123    alert_to = os.environ.get("ALERT_EMAIL_TO", smtp_user)124 125    solutions = """1263 WAYS TO FIX THIS:127 1281. RATE LIMIT EXHAUSTED129   → Wait 1 hour and re-run: python scripts/wiki_manager.py update130   → Increase delay: python scripts/batch_upgrade_wiki.py --delay 5131   → Switch provider by updating OPENROUTER_API_KEY or MISTRAL_API_KEY132 1332. API KEY EXPIRED / INVALID134   → OpenRouter: https://openrouter.ai/keys135   → Mistral: https://console.mistral.ai/api-keys136   → Gemini: https://aistudio.google.com/app/apikey137   → Update keys in chatbot_service/.env and GitHub Secrets138 1393. SERVICE OUTAGE140   → Check: https://status.openrouter.ai | https://status.mistral.ai141   → AST stubs remain functional as fallback documentation142   → Re-run later: python scripts/wiki_manager.py update143"""144 145    body = f"""SafeVixAI Wiki Manager — Alert146 147ISSUE: {subject}148TIME: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}149 150DETAILS:151{details}152 153CONTEXT:154{context}155{solutions}"""156 157    if smtp_user and smtp_pass:158        try:159            msg = MIMEText(body)160            msg["Subject"] = f"[SafeVixAI Wiki] {subject}"161            msg["From"] = smtp_user162            msg["To"] = alert_to163            with smtplib.SMTP("smtp.gmail.com", 587) as s:164                s.starttls()165                s.login(smtp_user, smtp_pass)166                s.send_message(msg)167            print(f"  📧 Alert emailed to {alert_to}")168        except Exception as e:169            print(f"  📧 Email failed ({e}), printing to console:")170            print(body)171    else:172        print(f"\n  ⚠️  ALERT: {subject}")173        print(f"  {details}")174        print(solutions)175 176 177# ══════════════════════════════════════════════════════════════════════════════178#  LLM PROVIDER179# ══════════════════════════════════════════════════════════════════════════════180 181class LLMProvider:182    """Multi-provider LLM with automatic fallback chain.183 184    Priority: GitHub Models (free via Student Pack) → OpenRouter → Mistral → Gemini.185    Handles 429 rate limits with exponential backoff + provider switching.186    """187 188    def __init__(self):189        self.github_token = os.environ.get("GITHUB_TOKEN", "")190        self.openrouter_key = os.environ.get("OPENROUTER_API_KEY", "")191        self.mistral_key = os.environ.get("MISTRAL_API_KEY", "")192        self.google_key = os.environ.get("GOOGLE_API_KEY", "")193        self._providers = []194        self.provider = None195        self._build_chain()196 197    def _build_chain(self):198        # GitHub Models is primary (free with GitHub Student Developer Pack)199        if self.github_token:200            self._providers.append("github-models")201        if self.openrouter_key:202            self._providers.append("openrouter")203        if self.mistral_key:204            self._providers.append("mistral")205        if self.google_key:206            self._providers.append("gemini")207 208        if self._providers:209            self.provider = self._providers[0]210            names = {211                "github-models": "GitHub Models (GPT-4o-mini)",212                "openrouter": "OpenRouter/Gemini",213                "mistral": "Mistral",214                "gemini": "Gemini Direct",215            }216            chain = " -> ".join(names.get(p, p) for p in self._providers)217            print(f"  LLM chain: {chain}")218        else:219            print("  LLM: None (will use AST-based stubs)")220 221    def generate(self, prompt, max_tokens=2048):222        """Generate text, trying each provider in fallback chain with retries."""223        if not self._providers:224            return None225 226        for provider in self._providers:227            result = self._try_provider(provider, prompt, max_tokens)228            if result:229                self.provider = provider230                return result231 232        print("    All LLM providers exhausted")233        return None234 235    def _try_provider(self, provider, prompt, max_tokens, max_retries=3):236        """Try a single provider with retry + exponential backoff on 429."""237        caller = {238            "github-models": self._call_github_models,239            "openrouter": self._call_openrouter,240            "mistral": self._call_mistral,241            "gemini": self._call_gemini,242        }.get(provider)243        if not caller:244            return None245 246        for attempt in range(max_retries):247            try:248                return caller(prompt, max_tokens)249            except HTTPError as e:250                if e.code == 429:251                    wait = (2 ** attempt) * 5  # 5s, 10s, 20s252                    print(f"    {provider} rate-limited (429). Retry {attempt+1}/{max_retries} in {wait}s...")253                    time.sleep(wait)254                else:255                    print(f"    {provider} HTTP {e.code}: {e.reason}")256                    return None257            except Exception as e:258                print(f"    {provider} error: {e}")259                return None260 261        print(f"    {provider} exhausted after {max_retries} retries")262        return None263 264    def _call_github_models(self, prompt, max_tokens):265        """Call GitHub Models API (free with GitHub Student Developer Pack).266 267        Uses GPT-4o-mini via Azure-hosted inference endpoint.268        Supports GPT-4o, GPT-4o-mini, Llama, Mistral, and more.269        """270        body = json.dumps({271            "model": "gpt-4o-mini",272            "messages": [{"role": "user", "content": prompt}],273            "max_tokens": max_tokens, "temperature": 0.3274        }).encode("utf-8")275        req = Request("https://models.inference.ai.azure.com/chat/completions", data=body, headers={276            "Content-Type": "application/json",277            "Authorization": f"Bearer {self.github_token}"278        })279        resp = urlopen(req, timeout=90)280        data = json.loads(resp.read().decode("utf-8"))281        return data["choices"][0]["message"]["content"]282 283    def _call_openrouter(self, prompt, max_tokens):284        body = json.dumps({285            "model": "google/gemini-2.0-flash-lite-001",286            "messages": [{"role": "user", "content": prompt}],287            "max_tokens": max_tokens, "temperature": 0.3288        }).encode("utf-8")289        req = Request("https://openrouter.ai/api/v1/chat/completions", data=body, headers={290            "Content-Type": "application/json",291            "Authorization": f"Bearer {self.openrouter_key}"292        })293        resp = urlopen(req, timeout=60)294        data = json.loads(resp.read().decode("utf-8"))295        return data["choices"][0]["message"]["content"]296 297    def _call_mistral(self, prompt, max_tokens):298        body = json.dumps({299            "model": "mistral-small-latest",300            "messages": [{"role": "user", "content": prompt}],301            "max_tokens": max_tokens, "temperature": 0.3302        }).encode("utf-8")303        req = Request("https://api.mistral.ai/v1/chat/completions", data=body, headers={304            "Content-Type": "application/json",305            "Authorization": f"Bearer {self.mistral_key}"306        })307        resp = urlopen(req, timeout=60)308        data = json.loads(resp.read().decode("utf-8"))309        return data["choices"][0]["message"]["content"]310 311    def _call_gemini(self, prompt, max_tokens):312        url = f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash-lite:generateContent?key={self.google_key}"313        body = json.dumps({314            "contents": [{"parts": [{"text": prompt}]}],315            "generationConfig": {"maxOutputTokens": max_tokens, "temperature": 0.3}316        }).encode("utf-8")317        req = Request(url, data=body, headers={"Content-Type": "application/json"})318        resp = urlopen(req, timeout=60)319        data = json.loads(resp.read().decode("utf-8"))320        return data["candidates"][0]["content"]["parts"][0]["text"]321 322 323# ══════════════════════════════════════════════════════════════════════════════324#  CODE DISCOVERY325# ══════════════════════════════════════════════════════════════════════════════326 327def discover_code_modules():328    modules = {}329    for code_dir, wiki_section in MODULE_MAP.items():330        full_dir = ROOT / code_dir.replace("/", os.sep)331        if not full_dir.exists():332            continue333        for f in full_dir.rglob("*"):334            if (f.is_file() and f.suffix in CODE_EXTENSIONS335                    and not any(s in str(f) for s in SKIP)336                    and f.stem not in SKIP):337                modules[str(f.relative_to(ROOT))] = {338                    "name": f.stem,339                    "section": wiki_section,340                    "path": str(f.relative_to(ROOT)),341                    "ext": f.suffix,342                }343    return modules344 345 346def discover_wiki_topics():347    topics = set()348    if not WIKI_CONTENT.exists():349        return topics350    for f in WIKI_CONTENT.rglob("*.md"):351        topics.add(f.stem.lower().replace(" ", "_").replace("-", "_"))352        try:353            text = f.read_text(encoding="utf-8")[:2000].lower()354            for m in re.findall(r'`(\w+(?:[_-]\w+)+)`', text):355                topics.add(m.replace("-", "_"))356        except Exception:357            pass358    return topics359 360 361def check_coverage(modules, topics):362    covered, uncovered = [], []363    for path, info in modules.items():364        key = info["name"].lower().replace("-", "_")365        if any(key in t or t in key for t in topics):366            covered.append(info)367        else:368            uncovered.append(info)369    return covered, uncovered370 371 372def read_source_code(filepath, max_lines=150):373    """Read source code, truncating if too long."""374    try:375        full_path = ROOT / filepath376        lines = full_path.read_text(encoding="utf-8", errors="ignore").split("\n")377        if len(lines) > max_lines:378            return "\n".join(lines[:max_lines]) + f"\n\n# ... truncated ({len(lines)} total lines)"379        return "\n".join(lines)380    except Exception:381        return ""382 383 384# ══════════════════════════════════════════════════════════════════════════════385#  DOCUMENTATION GENERATION386# ══════════════════════════════════════════════════════════════════════════════387 388def build_llm_prompt(info, source_code):389    """Build prompt for LLM to generate wiki documentation."""390    return f"""You are a technical documentation writer for SafeVixAI, an AI-powered road safety platform.391 392Generate a comprehensive wiki page in Markdown for the following module.393 394## Context395- Project: SafeVixAI — IIT Madras Road Safety Hackathon 2026396- Module: `{info['name']}{info['ext']}`397- Section: {info['section']}398- File: `{info['path']}`399- Platform uses: 9 LLM providers (Groq, Gemini, Cerebras, etc.), Supabase Auth, Next.js frontend, FastAPI backend400- Embeddings: LocalHashEmbeddingFunction (zero-dependency, SHA-256 based)401- Auth: Supabase Auth with JWT (no demo credentials)402 403## Source Code404```{info['ext'].lstrip('.')}405{source_code}406```407 408## Required Output Format409Generate a complete wiki page with these sections:4101. **Title** (# heading)4112. **Overview** — What this module does and why it exists (2-3 sentences)4123. **Architecture** — Where it fits in the system, include a mermaid flowchart showing data flow4134. **Key Classes/Functions** — Table with name, parameters, return type, description4145. **Dependencies** — What it imports/uses4156. **Configuration** — Any env vars, constants, or config needed4167. **Usage Examples** — Real code examples4178. **Error Handling** — How errors are managed4189. **Related Modules** — Links to related files419 420Rules:421- Be specific to THIS code, not generic422- Use actual function/class names from the source423- Include actual parameter types424- Keep it concise but complete425- Use tables for structured data426- No placeholder TODOs427- Output ONLY the markdown, no preamble428- Include at least one mermaid diagram (```mermaid) showing the module's data flow or class relationships429- Ensure mermaid syntax is valid: quote labels with special chars, no HTML tags in labels430"""431 432 433def generate_ast_stub(info):434    """Fallback: Generate stub using AST analysis (no LLM needed)."""435    name = info["name"]436    title = name.replace("_", " ").replace("-", " ").title()437    ext = info["ext"]438    lang = "python" if ext == ".py" else "typescript"439    now = datetime.now().strftime("%Y-%m-%d")440    source = info["path"]441    full_path = ROOT / source442 443    # Extract info from source444    docstring, classes, functions, imports = "", [], [], []445    try:446        text = full_path.read_text(encoding="utf-8", errors="ignore")447        if ext == ".py":448            try:449                tree = ast.parse(text)450                docstring = ast.get_docstring(tree) or ""451                for node in ast.walk(tree):452                    if isinstance(node, ast.ClassDef):453                        classes.append(node.name)454                    elif isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):455                        if not node.name.startswith("_"):456                            functions.append(node.name)457            except SyntaxError:458                pass459            for m in re.finditer(r'^(?:from|import)\s+(\w+)', text, re.MULTILINE):460                imp = m.group(1)461                if imp not in ("os", "sys", "re", "json", "typing", "datetime", "pathlib"):462                    imports.append(imp)463        else:464            for m in re.finditer(r'(?:export\s+)?(?:class|interface)\s+(\w+)', text):465                classes.append(m.group(1))466            for m in re.finditer(r'(?:export\s+)?(?:async\s+)?function\s+(\w+)', text):467                functions.append(m.group(1))468            for m in re.finditer(r'export\s+(?:const|let)\s+(\w+)\s*=', text):469                functions.append(m.group(1))470            for m in re.finditer(r"from ['\"]([^'\"]+)['\"]", text):471                if not m.group(1).startswith("."):472                    imports.append(m.group(1))473    except Exception:474        pass475 476    desc = docstring.split("\n")[0] if docstring else f"{title} module for the {info['section']} subsystem."477    sections = [f"# {title}\n", f"> Source: `{source}` | Generated: {now}\n", f"## Overview\n\n{desc}\n"]478 479    if classes:480        sections.append("## Classes\n")481        sections.append("| Class | Description |")482        sections.append("|---|---|")483        for c in classes[:10]:484            sections.append(f"| `{c}` | {c.replace('_',' ').title()} |")485        sections.append("")486 487    if functions:488        sections.append("## Key Functions\n")489        sections.append("| Function | Description |")490        sections.append("|---|---|")491        for fn in functions[:15]:492            sections.append(f"| `{fn}()` | {fn.replace('_',' ').title()} |")493        sections.append("")494 495    if imports:496        sections.append("## Dependencies\n")497        for imp in sorted(set(imports))[:10]:498            sections.append(f"- `{imp}`")499        sections.append("")500 501    sections.append(f"\n## File Location\n\n```\n{source}\n```\n")502    return "\n".join(sections)503 504 505def review_generated_doc(content, info, source_code, llm):506    """Self-review: validate generated doc matches source code.507 508    Checks:509    1. Function/class names in doc actually exist in source510    2. Mermaid diagrams have valid syntax (no unclosed blocks, no HTML tags)511    3. No hallucinated API endpoints or dependencies512    Returns (is_valid, issues) tuple.513    """514    issues = []515 516    # ── Check mermaid syntax ────────────────────────────────────────────517    import re as _re518    mermaid_blocks = _re.findall(r'```mermaid\s*\n(.*?)```', content, _re.DOTALL)519    for i, block in enumerate(mermaid_blocks):520        # Common mermaid errors521        if block.count('(') != block.count(')'):522            issues.append(f"Mermaid block {i+1}: unbalanced parentheses")523        if block.count('[') != block.count(']'):524            issues.append(f"Mermaid block {i+1}: unbalanced brackets")525        if '<br>' in block and '<br/>' not in block:526            issues.append(f"Mermaid block {i+1}: use <br/> not <br>")527 528    # ── Check function/class name accuracy ──────────────────────────────529    if source_code:530        ext = info.get("ext", ".py")531        if ext == ".py":532            try:533                tree = ast.parse(source_code)534                real_names = set()535                for node in ast.walk(tree):536                    if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)):537                        real_names.add(node.name)538                    elif isinstance(node, ast.ClassDef):539                        real_names.add(node.name)540 541                # Find backtick-quoted names in doc that look like function/class refs542                doc_refs = set(_re.findall(r'`(\w+)\(`', content))543                doc_refs |= set(_re.findall(r'`class\s+(\w+)`', content))544                doc_refs -= {"self", "cls", "str", "int", "dict", "list", "bool", "None",545                              "True", "False", "print", "len", "range", "type", "open",546                              "Exception", "ValueError", "TypeError", "KeyError"}547                # Ignore names that match (they're correct)548                hallucinated = doc_refs - real_names549                if hallucinated and len(hallucinated) > 3:550                    issues.append(f"Possibly hallucinated refs: {', '.join(sorted(hallucinated)[:5])}")551            except SyntaxError:552                pass  # Can't parse — skip name check553 554    # ── Minimal quality check ───────────────────────────────────────────555    if content.count("#") < 2:556        issues.append("Missing headings — too few # markers")557    if "TODO" in content or "PLACEHOLDER" in content:558        issues.append("Contains TODO/PLACEHOLDER markers")559 560    return len(issues) == 0, issues561 562 563def generate_doc(info, llm):564    """Generate documentation for a module using LLM with AST fallback.565 566    After LLM generation, runs a self-review to validate accuracy:567    - Checks function/class names match actual source code568    - Validates mermaid diagram syntax569    - Falls back to AST stub if review finds critical issues570    """571    source_code = read_source_code(info["path"])572 573    if llm.provider and source_code:574        prompt = build_llm_prompt(info, source_code)575        result = llm.generate(prompt, max_tokens=4096)576        if result and len(result) > 100:577            # Cap output to prevent bloated wiki pages578            if len(result) > 8000:579                result = result[:8000].rsplit("\n", 1)[0] + "\n"580 581            # ── Self-review generated content ───────────────────────582            is_valid, issues = review_generated_doc(result, info, source_code, llm)583            if not is_valid:584                name = info["name"]585                print(f"    ⚠ Review issues for {name}: {'; '.join(issues[:3])}")586                # If issues are minor (< 3), keep the doc but log warning587                # If critical (> 3 issues), fall through to AST588                if len(issues) > 3:589                    print(f"    ✘ Rejecting LLM output for {name} — falling back to AST")590                    return generate_ast_stub(info)591 592            return result593 594    # Fallback to AST595    return generate_ast_stub(info)596 597 598# ══════════════════════════════════════════════════════════════════════════════599#  CHECK MODE600# ══════════════════════════════════════════════════════════════════════════════601 602def check_staleness():603    stale_files = []604    for f in WIKI_CONTENT.rglob("*.md"):605        try:606            text = f.read_text(encoding="utf-8")607            issues = []608            for pat, _, desc in STALE_FIXES:609                for i, line in enumerate(text.split("\n"), 1):610                    if not is_safe_line(line) and re.search(pat, line, re.IGNORECASE):611                        issues.append((i, desc))612            if issues:613                stale_files.append((f.relative_to(ROOT), issues))614        except Exception:615            pass616    return stale_files617 618 619def run_check():620    print("=" * 60)621    print("SafeVixAI Wiki — Check Report")622    print("=" * 60)623 624    modules = discover_code_modules()625    topics = discover_wiki_topics()626    covered, uncovered = check_coverage(modules, topics)627    pct = len(covered) / len(modules) * 100 if modules else 100628 629    print(f"\n--- Coverage: {len(covered)}/{len(modules)} ({pct:.0f}%) ---")630    if uncovered:631        by_section = {}632        for info in uncovered:633            by_section.setdefault(info["section"], []).append(info)634        for section, items in sorted(by_section.items()):635            print(f"\n  [{section}] ({len(items)} undocumented)")636            for info in items[:5]:637                print(f"    - {info['name']}{info['ext']}")638            if len(items) > 5:639                print(f"    ... and {len(items) - 5} more")640 641    stale = check_staleness()642    print(f"\n--- Staleness: {len(stale)} files with stale refs ---")643    for fpath, issues in stale[:10]:644        print(f"  {fpath.name}: {len(issues)} issues")645 646    wiki_count = sum(1 for _ in WIKI_CONTENT.rglob("*.md")) if WIKI_CONTENT.exists() else 0647    print(f"\n--- Wiki: {wiki_count} content files ---")648 649    # Check docs/ alignment650    docs_md = list(DOCS_DIR.glob("*.md"))651    print(f"--- docs/ folder: {len(docs_md)} files ---")652 653    print(f"{'=' * 60}\n")654    return len(uncovered), len(stale)655 656 657# ══════════════════════════════════════════════════════════════════════════════658#  FIX MODE659# ══════════════════════════════════════════════════════════════════════════════660 661def run_fix():662    print("=" * 60)663    print("SafeVixAI Wiki — Fixing Stale References")664    print("=" * 60)665 666    total_fixes = 0667    files_fixed = 0668 669    # Fix wiki content670    for f in sorted(WIKI_CONTENT.rglob("*.md")):671        try:672            text = f.read_text(encoding="utf-8")673            original = text674            new_lines = []675            for line in text.split("\n"):676                if is_safe_line(line):677                    new_lines.append(line)678                    continue679                new_line = line680                for pat, repl, _ in STALE_FIXES:681                    new_line = re.sub(pat, repl, new_line, flags=re.IGNORECASE)682                new_lines.append(new_line)683            new_text = "\n".join(new_lines)684            if new_text != original:685                f.write_text(new_text, encoding="utf-8")686                count = sum(1 for o, n in zip(original.split("\n"), new_text.split("\n")) if o != n)687                total_fixes += count688                files_fixed += 1689                print(f"  Fixed: {f.name} ({count} lines)")690        except Exception as e:691            print(f"  Error: {f.name}: {e}")692 693    # Also fix docs/ folder694    for f in sorted(DOCS_DIR.glob("*.md")):695        if f.parent.name == "wiki":696            continue697        try:698            text = f.read_text(encoding="utf-8")699            original = text700            new_lines = []701            for line in text.split("\n"):702                if is_safe_line(line):703                    new_lines.append(line)704                    continue705                new_line = line706                for pat, repl, _ in STALE_FIXES:707                    new_line = re.sub(pat, repl, new_line, flags=re.IGNORECASE)708                new_lines.append(new_line)709            new_text = "\n".join(new_lines)710            if new_text != original:711                f.write_text(new_text, encoding="utf-8")712                count = sum(1 for o, n in zip(original.split("\n"), new_text.split("\n")) if o != n)713                total_fixes += count714                files_fixed += 1715                print(f"  Fixed: docs/{f.name} ({count} lines)")716        except Exception as e:717            pass718 719    # Fix root MD files720    for name in ["AGENTS.md", "README.md", "DESIGN.md", "SETUP.md", "SKILL.md"]:721        f = ROOT / name722        if not f.exists():723            continue724        try:725            text = f.read_text(encoding="utf-8")726            original = text727            new_lines = []728            for line in text.split("\n"):729                if is_safe_line(line):730                    new_lines.append(line)731                    continue732                new_line = line733                for pat, repl, _ in STALE_FIXES:734                    new_line = re.sub(pat, repl, new_line, flags=re.IGNORECASE)735                new_lines.append(new_line)736            new_text = "\n".join(new_lines)737            if new_text != original:738                f.write_text(new_text, encoding="utf-8")739                count = sum(1 for o, n in zip(original.split("\n"), new_text.split("\n")) if o != n)740                total_fixes += count741                files_fixed += 1742                print(f"  Fixed: {name} ({count} lines)")743        except Exception as e:744            pass745 746    print(f"\n  Total: {total_fixes} fixes across {files_fixed} files")747    return total_fixes748 749 750# ══════════════════════════════════════════════════════════════════════════════751#  GENERATE MODE752# ══════════════════════════════════════════════════════════════════════════════753 754def run_generate():755    print("=" * 60)756    print("SafeVixAI Wiki — Generating Documentation")757    print("=" * 60)758 759    llm = LLMProvider()760 761    modules = discover_code_modules()762    topics = discover_wiki_topics()763    _, uncovered = check_coverage(modules, topics)764 765    if not uncovered:766        print("\n  All code modules are documented!")767        return 0768 769    print(f"\n  Found {len(uncovered)} undocumented modules. Generating...")770 771    created = 0772    for info in sorted(uncovered, key=lambda x: (x["section"], x["name"])):773        section_dir = WIKI_CONTENT / info["section"]774        section_dir.mkdir(parents=True, exist_ok=True)775 776        title = info["name"].replace("_", " ").replace("-", " ").title()777        stub_path = section_dir / f"{title}.md"778 779        if stub_path.exists():780            continue781 782        content = generate_doc(info, llm)783        stub_path.write_text(content, encoding="utf-8")784        created += 1785 786        method = "LLM" if llm.provider else "AST"787        print(f"  [{method}] Created: {info['section']}/{title}.md")788 789        # Rate limit per provider790        if llm.provider in ("gemini",):791            time.sleep(4.5)792        elif llm.provider in ("openrouter", "mistral"):793            time.sleep(2)794 795    print(f"\n  Total: {created} wiki files created")796    return created797 798 799# ══════════════════════════════════════════════════════════════════════════════800#  UPDATE MODE — Re-generate outdated docs801# ══════════════════════════════════════════════════════════════════════════════802 803def run_update():804    """Check existing wiki docs against source code and update if outdated."""805    print("=" * 60)806    print("SafeVixAI Wiki — Updating Outdated Documentation")807    print("=" * 60)808 809    llm = LLMProvider()810    if not llm.provider:811        print("  No LLM available — update mode requires an API key.")812        return 0813 814    modules = discover_code_modules()815    updated = 0816    failed = 0817    consecutive_fails = 0818 819    for path, info in sorted(modules.items()):820        title = info["name"].replace("_", " ").replace("-", " ").title()821        wiki_path = WIKI_CONTENT / info["section"] / f"{title}.md"822 823        if not wiki_path.exists():824            continue825 826        wiki_text = wiki_path.read_text(encoding="utf-8")827 828        # Check if wiki is auto-generated stub (needs upgrade)829        if "Auto-generated:" in wiki_text:830            source_code = read_source_code(info["path"])831            if source_code:832                content = generate_doc(info, llm)833                if content and len(content) > 100:834                    wiki_path.write_text(content, encoding="utf-8")835                    updated += 1836                    consecutive_fails = 0837                    print(f"  Updated: {info['section']}/{title}.md")838 839                    if llm.provider in ("gemini",):840                        time.sleep(4.5)841                    elif llm.provider in ("openrouter", "mistral"):842                        time.sleep(2)843                else:844                    failed += 1845                    consecutive_fails += 1846                    print(f"  FAILED: {info['section']}/{title}.md")847 848                    if consecutive_fails >= 5:849                        send_alert(850                            "Wiki update stopped — 5 consecutive LLM failures",851                            f"Failed at: {info['section']}/{title}.md\n"852                            f"Updated {updated} files before failure, {failed} total failures.",853                            f"Provider chain: {', '.join(llm._providers)}"854                        )855                        if get_alert_service:856                            try:857                                get_alert_service().alert_wiki_generation_failed(858                                    module_name=f"{info['section']}/{title}.md",859                                    consecutive_fails=consecutive_fails,860                                    error_msg=f"Updated {updated} files before failure. Chain: {', '.join(llm._providers)}"861                                )862                            except Exception:863                                pass864                        break865 866    print(f"\n  Total: {updated} updated, {failed} failed")867    return updated868 869 870# ══════════════════════════════════════════════════════════════════════════════871#  FULL MODE872# ══════════════════════════════════════════════════════════════════════════════873 874def run_review():875    """Standalone review mode: verify all existing wiki docs for quality."""876    print("=" * 60)877    print("SafeVixAI Wiki — Self-Review (Quality + Mermaid + Codebase)")878    print("=" * 60)879 880    if not WIKI_CONTENT.exists():881        print("  No wiki content found.")882        return 0883 884    modules = discover_code_modules()885    total_docs = 0886    total_issues = 0887    mermaid_errors = 0888    hallucinated = 0889    quality_fail = 0890 891    for f in sorted(WIKI_CONTENT.rglob("*.md")):892        total_docs += 1893        text = f.read_text(encoding="utf-8")894 895        stem = f.stem.lower().replace(" ", "_").replace("-", "_")896        matched_info = None897        matched_source = ""898        for path, info in modules.items():899            if info["name"].lower().replace("-", "_") == stem:900                matched_info = info901                matched_source = read_source_code(info["path"])902                break903 904        if not matched_info:905            matched_info = {"name": f.stem, "ext": ".py", "path": "", "section": ""}906 907        is_valid, issues = review_generated_doc(text, matched_info, matched_source, None)908        if not is_valid:909            total_issues += 1910            for issue in issues:911                if "Mermaid" in issue:912                    mermaid_errors += 1913                elif "hallucinated" in issue:914                    hallucinated += 1915                else:916                    quality_fail += 1917            if issues:918                print(f"  Warning: {f.name}: {'; '.join(issues[:2])}")919 920    print(f"\n  {'=' * 50}")921    print(f"  REVIEW SUMMARY")922    print(f"  Total docs reviewed: {total_docs}")923    print(f"  Docs with issues: {total_issues}")924    print(f"    Mermaid syntax: {mermaid_errors}")925    print(f"    Hallucinated refs: {hallucinated}")926    print(f"    Quality fails: {quality_fail}")927    print(f"  Clean docs: {total_docs - total_issues}")928    print(f"  {'=' * 50}")929    return total_issues930 931 932def run_full():933    print("=" * 60)934    print("SafeVixAI Wiki — Full Update")935    print("=" * 60)936 937    fixes = run_fix()938    print()939    created = run_generate()940    print()941    review_issues = run_review()942 943    print(f"\n{'=' * 60}")944    print(f"Summary: {fixes} stale fixes + {created} new docs + {review_issues} review issues")945    print(f"{'=' * 60}")946    return fixes + created947 948 949# ══════════════════════════════════════════════════════════════════════════════950#  CLI951# ══════════════════════════════════════════════════════════════════════════════952 953def main():954    mode = sys.argv[1] if len(sys.argv) > 1 else "check"955 956    if mode == "check":957        uncov, stale = run_check()958        sys.exit(1 if stale > 0 else 0)959    elif mode == "fix":960        run_fix()961    elif mode == "generate":962        run_generate()963    elif mode == "update":964        run_update()965    elif mode == "review":966        issues = run_review()967        sys.exit(1 if issues > 0 else 0)968    elif mode == "full":969        run_full()970    else:971        print(f"Usage: python {sys.argv[0]} [check|fix|generate|update|review|full]")972        sys.exit(1)973 974 975if __name__ == "__main__":976    main()977