nothiro/cereal-applicant-api

1"""2GitHub service — Fetch repos + READMEs via Clerk OAuth, summarize with Haiku, index.3"""4 5import os6import logging7from datetime import datetime8 9import httpx10 11logger = logging.getLogger("cereal.services.github")12 13CLERK_API_URL = os.getenv("CLERK_API_URL", "https://api.clerk.com/v1")14CLERK_SECRET_KEY = os.getenv("CLERK_SECRET_KEY", "")15 16 17async def _get_all_github_tokens(user_id: str) -> list[dict]:18    """19    Return all GitHub tokens for this user from Clerk.20    Each dict: { "token": str, "login": str }21    """22    async with httpx.AsyncClient() as client:23        resp = await client.get(24            f"{CLERK_API_URL}/users/{user_id}/oauth_access_tokens/oauth_github",25            headers={"Authorization": f"Bearer {CLERK_SECRET_KEY}"},26        )27        resp.raise_for_status()28        tokens = resp.json()29        if not tokens:30            raise ValueError("No GitHub OAuth tokens found for user")31 32        results = []33        for t in tokens:34            token = t["token"]35            try:36                me = await client.get(37                    "https://api.github.com/user",38                    headers={"Authorization": f"Bearer {token}",39                             "Accept": "application/vnd.github.v3+json"},40                )41                login = me.json().get("login", "unknown")42            except Exception:43                login = "unknown"44            results.append({"token": token, "login": login})45        return results46 47 48async def fetch_and_index_github(user_id: str) -> dict:49    """Index repos from ALL connected GitHub accounts."""50    from services.elasticsearch import bulk_index, delete_user_docs51    from services.llm import summarize_readme52 53    accounts = await _get_all_github_tokens(user_id)54    logger.info(f"{len(accounts)} GitHub account(s) for {user_id}")55 56    await delete_user_docs("github_projects", user_id)57 58    all_docs = []59    now = datetime.utcnow().isoformat()60 61    async with httpx.AsyncClient() as client:62        for acct in accounts:63            token = acct["token"]64            login = acct["login"]65            repos = []66            page  = 167 68            while True:69                r = await client.get(70                    "https://api.github.com/user/repos",71                    headers={"Authorization": f"Bearer {token}",72                             "Accept": "application/vnd.github.v3+json"},73                    params={"sort":"updated","per_page":30,"page":page,"affiliation":"owner"},74                )75                r.raise_for_status()76                batch = r.json()77                if not batch: break78                repos.extend(batch)79                page += 180                if page > 5: break   # Cap at 150 repos per account81 82            for repo in repos:83                if repo.get("fork"): continue84                readme = ""85                try:86                    rr = await client.get(87                        f"https://api.github.com/repos/{repo['full_name']}/readme",88                        headers={"Authorization": f"Bearer {token}",89                                 "Accept": "application/vnd.github.v3.raw"},90                    )91                    if rr.status_code == 200: readme = rr.text92                except Exception: pass93 94                summary = ""95                if readme:96                    try: summary = await summarize_readme(readme, repo["name"])97                    except Exception: summary = readme[:500]98 99                all_docs.append({100                    "user_id":        user_id,101                    "github_account": login,   # track which account102                    "repo_name":      repo["name"],103                    "repo_url":       repo["html_url"],104                    "stars":          repo.get("stargazers_count", 0),105                    "language":       repo.get("language", ""),106                    "topics":         repo.get("topics", []),107                    "readme_summary": summary or f"{repo['name']}: {repo.get('description','')}",108                    "indexed_at":     now,109                })110 111        if all_docs:112            await bulk_index("github_projects", all_docs)113    logger.info(f"Indexed {len(all_docs)} repos across {len(accounts)} account(s)")114    return {"repo_count": len(all_docs), "account_count": len(accounts)}115