jblitzar/github-python

1#!/usr/bin/env python32"""3Create metadata table for the elaborated GitHub Python dataset.4 5This script parses the python_files_elaborated.txt file containing GitHub URLs6and extracts repository metadata (owner, repo name, file path).7It generates a CSV file with this information and prints statistics.8 9The elaborated dataset contains more files than the licensed subset and10may include repositories with various licenses (not just permissive ones).11"""12 13import csv14import os15import re16import pandas as pd17from collections import Counter, defaultdict18from tqdm import tqdm19from urllib.parse import urlparse20 21# Input and output files22ELABORATED_FILES_LIST = "python_files_elaborated.txt"23LICENSED_FILES_LIST = "python_files.txt"24OUTPUT_CSV = "python_files_elaborated_metadata.csv"25 26# Regular expression to parse GitHub raw URLs27# Format: https://raw.githubusercontent.com/OWNER/REPO/BRANCH/PATH28GITHUB_RAW_PATTERN = r"https://raw\.githubusercontent\.com/([^/]+)/([^/]+)/[^/]+/(.*)"29 30 31def parse_github_url(url):32    """33    Parse a GitHub raw URL to extract owner, repo name, and file path.34    35    Args:36        url (str): GitHub raw URL37        38    Returns:39        tuple: (owner, repo_name, file_path) or None if URL doesn't match pattern40    """41    match = re.match(GITHUB_RAW_PATTERN, url)42    if match:43        owner, repo_name, file_path = match.groups()44        return owner, repo_name, file_path45    return None46 47 48def create_metadata_table(file_list_path):49    """50    Create a metadata table from a list of GitHub URLs.51    52    Args:53        file_list_path (str): Path to file containing GitHub URLs54        55    Returns:56        list: List of dictionaries with metadata57    """58    metadata = []59    60    # Read URLs from file61    with open(file_list_path, "r") as f:62        urls = [line.strip() for line in f if line.strip()]63    64    print(f"Processing URLs from {file_list_path}...")65    66    # Parse each URL and extract metadata67    for url in tqdm(urls, desc="Parsing URLs"):68        parsed = parse_github_url(url)69        if parsed:70            owner, repo_name, file_path = parsed71            metadata.append({72                "owner": owner,73                "repo_name": repo_name,74                "file_path": file_path,75                "url": url76            })77    78    return metadata79 80 81def generate_statistics(metadata, dataset_name):82    """83    Generate and print statistics for the dataset.84    85    Args:86        metadata (list): List of dictionaries with metadata87        dataset_name (str): Name of the dataset for display88    """89    # Count unique repositories and owners90    repos = set((item["owner"], item["repo_name"]) for item in metadata)91    owners = set(item["owner"] for item in metadata)92    93    # Count files by repository94    repo_counts = Counter((item["owner"], item["repo_name"]) for item in metadata)95    top_repos = repo_counts.most_common(10)96    97    # Count files by owner98    owner_counts = Counter(item["owner"] for item in metadata)99    top_owners = owner_counts.most_common(5)100    101    # Count file extensions102    extensions = Counter(os.path.splitext(item["file_path"])[1] for item in metadata)103    104    # Print statistics105    print(f"\n=== {dataset_name} Statistics ===")106    print(f"Total files: {len(metadata)}")107    print(f"Unique repositories: {len(repos)}")108    print(f"Unique repository owners: {len(owners)}")109    110    print("\nTop 10 repositories by file count:")111    for (owner, repo), count in top_repos:112        print(f"  {owner}/{repo}: {count} files")113    114    print("\nFile extensions:")115    for ext, count in extensions.most_common():116        if ext:  # Skip empty extensions117            print(f"  {ext}: {count} files")118    119    print("\nTop 5 repository owners:")120    for owner, count in top_owners:121        print(f"  {owner}: {count} files")122    123    return {124        "total_files": len(metadata),125        "unique_repos": len(repos),126        "unique_owners": len(owners),127        "top_repos": top_repos,128        "top_owners": top_owners,129        "extensions": extensions130    }131 132 133def compare_datasets(elaborated_stats, licensed_stats):134    """135    Compare statistics between elaborated and licensed datasets.136    137    Args:138        elaborated_stats (dict): Statistics for elaborated dataset139        licensed_stats (dict): Statistics for licensed dataset140    """141    print("\n=== Dataset Comparison ===")142    print(f"Elaborated dataset: {elaborated_stats['total_files']} files")143    print(f"Licensed dataset: {licensed_stats['total_files']} files")144    print(f"Additional files in elaborated dataset: {elaborated_stats['total_files'] - licensed_stats['total_files']} files")145    146    # Calculate percentage increase147    pct_increase = ((elaborated_stats['total_files'] / licensed_stats['total_files']) - 1) * 100148    print(f"Percentage increase: {pct_increase:.1f}%")149    150    # Compare repositories151    print(f"\nElaborated dataset: {elaborated_stats['unique_repos']} repositories")152    print(f"Licensed dataset: {licensed_stats['unique_repos']} repositories")153    154    # Compare owners155    print(f"\nElaborated dataset: {elaborated_stats['unique_owners']} repository owners")156    print(f"Licensed dataset: {licensed_stats['unique_owners']} repository owners")157    158    # Find repositories unique to elaborated dataset159    elaborated_repos = set((owner, repo) for (owner, repo), _ in elaborated_stats['top_repos'])160    licensed_repos = set((owner, repo) for (owner, repo), _ in licensed_stats['top_repos'])161    unique_to_elaborated = elaborated_repos - licensed_repos162    163    if unique_to_elaborated:164        print("\nTop repositories unique to elaborated dataset:")165        for owner, repo in list(unique_to_elaborated)[:5]:166            print(f"  {owner}/{repo}")167 168 169def main():170    # Process elaborated dataset171    elaborated_metadata = create_metadata_table(ELABORATED_FILES_LIST)172    173    # Save to CSV174    with open(OUTPUT_CSV, "w", newline="") as f:175        writer = csv.DictWriter(f, fieldnames=["owner", "repo_name", "file_path", "url"], 176                               quoting=csv.QUOTE_MINIMAL)177        writer.writeheader()178        writer.writerows(elaborated_metadata)179    180    print(f"Metadata saved to {OUTPUT_CSV}")181    182    # Generate statistics for elaborated dataset183    elaborated_stats = generate_statistics(elaborated_metadata, "Elaborated Dataset")184    185    # Process licensed dataset for comparison186    if os.path.exists(LICENSED_FILES_LIST):187        licensed_metadata = create_metadata_table(LICENSED_FILES_LIST)188        licensed_stats = generate_statistics(licensed_metadata, "Licensed Dataset")189        190        # Compare datasets191        compare_datasets(elaborated_stats, licensed_stats)192    else:193        print(f"Warning: {LICENSED_FILES_LIST} not found. Cannot compare datasets.")194 195 196if __name__ == "__main__":197    main()198