Vedanshipanda/layer10-api

1import os2import requests3import json4import time5from dotenv import load_dotenv6 7load_dotenv()8TOKEN = os.getenv("GITHUB_TOKEN")9HEADERS = {"Authorization": f"token {TOKEN}"}10 11REPO_OWNER = "tiangolo"12REPO_NAME = "fastapi"13 14# 🚀 TARGET: Fetch 500 items to ensure ~500 nodes / 400 edges15TARGET_COUNT = 50016PER_PAGE = 100  # Max allowed by GitHub API per request17 18def fetch_issues_paginated():19    """Fetches issues/PRs recursively until TARGET_COUNT is met."""20    all_issues = []21    page = 122    23    print(f"🚀 Starting ingestion from {REPO_OWNER}/{REPO_NAME}...")24    25    while len(all_issues) < TARGET_COUNT:26        url = f"https://api.github.com/repos/{REPO_OWNER}/{REPO_NAME}/issues"27        params = {28            "state": "all",       29            "per_page": PER_PAGE,30            "sort": "comments",  # Sorting by 'comments' gets the most active/connected issues31            "direction": "desc",32            "page": page33        }34        35        print(f"📡 Fetching Page {page} (Current Total: {len(all_issues)})...")36        response = requests.get(url, headers=HEADERS, params=params)37        38        if response.status_code != 200:39            print(f"❌ Error: {response.status_code} - {response.text}")40            break41 42        batch = response.json()43        if not batch:44            break  # No more data available45            46        all_issues.extend(batch)47        page += 148        49        # Respect GitHub API rate limits50        time.sleep(1)51 52    # Trim to exact target53    return all_issues[:TARGET_COUNT]54 55def save_data(data):56    os.makedirs("data", exist_ok=True)57    filepath = os.path.join("data", "raw_corpus.json")58    with open(filepath, "w", encoding="utf-8") as f:59        json.dump(data, f, indent=4)60    print(f"✅ Successfully saved {len(data)} items to {filepath}")61 62if __name__ == "__main__":63    data = fetch_issues_paginated()64    if data:65        save_data(data)