jblitzar/github-python

1#!/usr/bin/env python32"""3Create a metadata table from GitHub Python file URLs.4 5This script processes the file URLs from python_files.txt and creates a tabular6CSV file with repository metadata including owner, name, file path, and URLs.7"""8 9import os10import re11import csv12import pandas as pd13from collections import Counter14from urllib.parse import urlparse15from tqdm import tqdm16 17 18def parse_github_url(url):19    """20    Parse a GitHub URL to extract repository owner, name, and file path.21    22    Handles both raw.githubusercontent.com and github.com URLs.23    24    Args:25        url (str): GitHub URL26        27    Returns:28        dict: Dictionary with repo_owner, repo_name, file_path, repo_url29    """30    url = url.strip()31    32    # Initialize default values33    result = {34        "repo_owner": "unknown",35        "repo_name": "unknown",36        "file_path": "",37        "file_url": url,38        "repo_url": ""39    }40    41    try:42        # Parse URL to get components43        parsed = urlparse(url)44        path_parts = parsed.path.strip('/').split('/')45        46        # Handle raw.githubusercontent.com URLs47        # Format: https://raw.githubusercontent.com/owner/repo/branch/path/to/file.py48        if 'raw.githubusercontent.com' in url:49            if len(path_parts) >= 3:50                result["repo_owner"] = path_parts[0]51                result["repo_name"] = path_parts[1]52                # Skip branch (path_parts[2]) and get the rest as file path53                result["file_path"] = '/'.join(path_parts[3:])54                result["repo_url"] = f"https://github.com/{path_parts[0]}/{path_parts[1]}"55        56        # Handle github.com URLs57        # Format: https://github.com/owner/repo/blob/branch/path/to/file.py58        elif 'github.com' in url:59            if len(path_parts) >= 4 and path_parts[2] == 'blob':60                result["repo_owner"] = path_parts[0]61                result["repo_name"] = path_parts[1]62                # Skip 'blob' and branch, get the rest as file path63                result["file_path"] = '/'.join(path_parts[4:])64                result["repo_url"] = f"https://github.com/{path_parts[0]}/{path_parts[1]}"65        66        return result67    68    except Exception as e:69        print(f"Error parsing URL {url}: {e}")70        return result71 72 73def process_file_urls(input_file, output_file):74    """75    Process GitHub file URLs and create a metadata CSV file.76    77    Args:78        input_file (str): Path to the file containing GitHub URLs79        output_file (str): Path to the output CSV file80    """81    print(f"Processing URLs from {input_file}...")82    83    # Read file URLs84    with open(input_file, 'r', encoding='utf-8') as f:85        urls = [line.strip() for line in f if line.strip()]86    87    # Parse each URL88    metadata = []89    for url in tqdm(urls, desc="Parsing URLs"):90        metadata.append(parse_github_url(url))91    92    # Convert to DataFrame93    df = pd.DataFrame(metadata)94    95    # Save to CSV96    # Use minimal quoting to remain compatible with the standard csv module97    df.to_csv(output_file, index=False, quoting=csv.QUOTE_MINIMAL)98    print(f"Metadata saved to {output_file}")99    100    # Print statistics101    unique_repos = df[['repo_owner', 'repo_name']].drop_duplicates()102    unique_owners = df['repo_owner'].nunique()103    104    print("\n=== Dataset Statistics ===")105    print(f"Total files: {len(df)}")106    print(f"Unique repositories: {len(unique_repos)}")107    print(f"Unique repository owners: {unique_owners}")108    109    # Top repositories by file count110    repo_counts = Counter(zip(df['repo_owner'], df['repo_name']))111    print("\nTop 10 repositories by file count:")112    for (owner, repo), count in repo_counts.most_common(10):113        print(f"  {owner}/{repo}: {count} files")114    115    # File extensions116    extensions = Counter([os.path.splitext(path)[1] for path in df['file_path'] if path])117    print("\nFile extensions:")118    for ext, count in extensions.most_common(5):119        print(f"  {ext or 'No extension'}: {count} files")120    121    # Repository owners with most repositories122    owner_repo_counts = Counter(df['repo_owner'])123    print("\nTop 5 repository owners:")124    for owner, count in owner_repo_counts.most_common(5):125        print(f"  {owner}: {count} files")126 127 128if __name__ == "__main__":129    input_file = "python_files.txt"130    output_file = "github_python_metadata.csv"131    132    # Check if input file exists133    if not os.path.exists(input_file):134        print(f"Error: Input file {input_file} not found.")135        print("Please make sure the file exists in the current directory.")136        exit(1)137    138    process_file_urls(input_file, output_file)139