CoolFace
Datasetpublic

taher-ghaleb/DEPosit

DEPosit — Replication Package DEPosit (Data Engineering Pipeline Repositories) is a dataset of open-source GitHub repositories in the data-engineering pipeline ecosystem. The dataset contains a master cohort of 1,952 repositories together with repository activity, commits, pull requests, issues, CI-service information, contributor metrics, and pipeline discovery and feature data. Dataset (Data/) The Hugging Face Hub exposes each heterogeneous CSV table as a… See the full description on the dataset page: https://huggingface.co/datasets/taher-ghaleb/DEPosit.

sourceHugging Facecc-by-4.0updated 1mo agoView on Hugging Face
1likes51downloads
github_tokens.py127 linesDownload Raw Back to Scripts
1"""GitHub PAT configuration for DEPosit collection scripts.
2
3Never commit personal access tokens. Provide credentials via:
4
5  1. Environment variables (recommended for CI)
6  2. A local ``.env`` file (copy from ``.env.example``; not committed)
7  3. ``--token`` / ``--tokens`` CLI flags where supported
8
9Variables:
10  GITHUB_TOKEN_SE4DE  — project-specific name (preferred)
11  GITHUB_TOKEN        — standard name
12  GITHUB_TOKENS       — comma-separated PATs for rate-limit rotation
13"""
14from __future__ import annotations
15
16import argparse
17import os
18import sys
19from pathlib import Path
20
21PACKAGE_ROOT = Path(__file__).resolve().parents[1]
22
23
24def load_dotenv(path: Path | None = None) -> None:
25    """Load KEY=VALUE pairs from .env into os.environ (does not override existing)."""
26    path = path or PACKAGE_ROOT / ".env"
27    if not path.is_file():
28        return
29    for raw in path.read_text(encoding="utf-8").splitlines():
30        line = raw.strip()
31        if not line or line.startswith("#") or "=" not in line:
32            continue
33        key, _, value = line.partition("=")
34        key = key.strip()
35        value = value.strip().strip("'").strip('"')
36        if key and key not in os.environ:
37            os.environ[key] = value
38
39
40def load_tokens() -> list[str]:
41    """Return all configured PATs (may be empty)."""
42    load_dotenv()
43    found: list[str] = []
44    for env in ("GITHUB_TOKEN_SE4DE", "GITHUB_TOKEN"):
45        value = os.environ.get(env, "").strip()
46        if value and value not in found:
47            found.append(value)
48    for part in os.environ.get("GITHUB_TOKENS", "").split(","):
49        part = part.strip()
50        if part and part not in found:
51            found.append(part)
52    return found
53
54
55def build_authorization_headers(tokens: list[str]) -> list[dict[str, str]]:
56    return [
57        {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}
58        for token in tokens
59    ]
60
61
62def resolve_token(cli_token: str | None = None, cli_tokens: str | None = None) -> str | None:
63    """Single PAT: CLI overrides environment."""
64    if cli_token and cli_token.strip():
65        return cli_token.strip()
66    if cli_tokens:
67        parts = [p.strip() for p in cli_tokens.split(",") if p.strip()]
68        if parts:
69            return parts[0]
70    tokens = load_tokens()
71    return tokens[0] if tokens else None
72
73
74def require_github_tokens(
75    cli_token: str | None = None,
76    cli_tokens: str | None = None,
77) -> list[str]:
78    """Return PAT list or exit with instructions."""
79    if cli_token and cli_token.strip():
80        return [cli_token.strip()]
81    if cli_tokens:
82        parts = [p.strip() for p in cli_tokens.split(",") if p.strip()]
83        if parts:
84            return parts
85    tokens = load_tokens()
86    if not tokens:
87        _exit_missing_token()
88    return tokens
89
90
91def primary_token(cli_token: str | None = None, cli_tokens: str | None = None) -> str:
92    token = resolve_token(cli_token, cli_tokens)
93    if not token:
94        _exit_missing_token()
95    return token
96
97
98def init_rest_client_auth(
99    cli_token: str | None = None,
100    cli_tokens: str | None = None,
101) -> tuple[list[str], list[dict[str, str]]]:
102    """Used by REST collection scripts with token rotation."""
103    tokens = require_github_tokens(cli_token, cli_tokens)
104    return tokens, build_authorization_headers(tokens)
105
106
107def add_github_token_args(parser: argparse.ArgumentParser) -> None:
108    group = parser.add_argument_group("GitHub authentication")
109    group.add_argument(
110        "--token",
111        metavar="PAT",
112        help="GitHub personal access token (overrides environment)",
113    )
114    group.add_argument(
115        "--tokens",
116        metavar="PAT1,PAT2",
117        help="Comma-separated PATs for rate-limit rotation (overrides environment)",
118    )
119
120
121def _exit_missing_token() -> None:
122    print("GitHub token required for API collection.", file=sys.stderr)
123    print("  Option A: copy .env.example to .env and set GITHUB_TOKEN_SE4DE", file=sys.stderr)
124    print("  Option B: $env:GITHUB_TOKEN_SE4DE = '<your-pat>'  (PowerShell)", file=sys.stderr)
125    print("  Option C: pass --token <your-pat> on scripts that support it", file=sys.stderr)
126    sys.exit(1)
127