taher-ghaleb/DEPosit
DEPosit — Replication Package DEPosit (Data Engineering Pipeline Repositories) is a dataset of open-source GitHub repositories in the data-engineering pipeline ecosystem. The dataset contains a master cohort of 1,952 repositories together with repository activity, commits, pull requests, issues, CI-service information, contributor metrics, and pipeline discovery and feature data. Dataset (Data/) The Hugging Face Hub exposes each heterogeneous CSV table as a… See the full description on the dataset page: https://huggingface.co/datasets/taher-ghaleb/DEPosit.
151
1"""GitHub PAT configuration for DEPosit collection scripts.
2
3Never commit personal access tokens. Provide credentials via:
4
5 1. Environment variables (recommended for CI)
6 2. A local ``.env`` file (copy from ``.env.example``; not committed)
7 3. ``--token`` / ``--tokens`` CLI flags where supported
8
9Variables:
10 GITHUB_TOKEN_SE4DE — project-specific name (preferred)
11 GITHUB_TOKEN — standard name
12 GITHUB_TOKENS — comma-separated PATs for rate-limit rotation
13"""
14from __future__ import annotations
15
16import argparse
17import os
18import sys
19from pathlib import Path
20
21PACKAGE_ROOT = Path(__file__).resolve().parents[1]
22
23
24def load_dotenv(path: Path | None = None) -> None:
25 """Load KEY=VALUE pairs from .env into os.environ (does not override existing)."""
26 path = path or PACKAGE_ROOT / ".env"
27 if not path.is_file():
28 return
29 for raw in path.read_text(encoding="utf-8").splitlines():
30 line = raw.strip()
31 if not line or line.startswith("#") or "=" not in line:
32 continue
33 key, _, value = line.partition("=")
34 key = key.strip()
35 value = value.strip().strip("'").strip('"')
36 if key and key not in os.environ:
37 os.environ[key] = value
38
39
40def load_tokens() -> list[str]:
41 """Return all configured PATs (may be empty)."""
42 load_dotenv()
43 found: list[str] = []
44 for env in ("GITHUB_TOKEN_SE4DE", "GITHUB_TOKEN"):
45 value = os.environ.get(env, "").strip()
46 if value and value not in found:
47 found.append(value)
48 for part in os.environ.get("GITHUB_TOKENS", "").split(","):
49 part = part.strip()
50 if part and part not in found:
51 found.append(part)
52 return found
53
54
55def build_authorization_headers(tokens: list[str]) -> list[dict[str, str]]:
56 return [
57 {"Accept": "application/vnd.github+json", "Authorization": f"Bearer {token}"}
58 for token in tokens
59 ]
60
61
62def resolve_token(cli_token: str | None = None, cli_tokens: str | None = None) -> str | None:
63 """Single PAT: CLI overrides environment."""
64 if cli_token and cli_token.strip():
65 return cli_token.strip()
66 if cli_tokens:
67 parts = [p.strip() for p in cli_tokens.split(",") if p.strip()]
68 if parts:
69 return parts[0]
70 tokens = load_tokens()
71 return tokens[0] if tokens else None
72
73
74def require_github_tokens(
75 cli_token: str | None = None,
76 cli_tokens: str | None = None,
77) -> list[str]:
78 """Return PAT list or exit with instructions."""
79 if cli_token and cli_token.strip():
80 return [cli_token.strip()]
81 if cli_tokens:
82 parts = [p.strip() for p in cli_tokens.split(",") if p.strip()]
83 if parts:
84 return parts
85 tokens = load_tokens()
86 if not tokens:
87 _exit_missing_token()
88 return tokens
89
90
91def primary_token(cli_token: str | None = None, cli_tokens: str | None = None) -> str:
92 token = resolve_token(cli_token, cli_tokens)
93 if not token:
94 _exit_missing_token()
95 return token
96
97
98def init_rest_client_auth(
99 cli_token: str | None = None,
100 cli_tokens: str | None = None,
101) -> tuple[list[str], list[dict[str, str]]]:
102 """Used by REST collection scripts with token rotation."""
103 tokens = require_github_tokens(cli_token, cli_tokens)
104 return tokens, build_authorization_headers(tokens)
105
106
107def add_github_token_args(parser: argparse.ArgumentParser) -> None:
108 group = parser.add_argument_group("GitHub authentication")
109 group.add_argument(
110 "--token",
111 metavar="PAT",
112 help="GitHub personal access token (overrides environment)",
113 )
114 group.add_argument(
115 "--tokens",
116 metavar="PAT1,PAT2",
117 help="Comma-separated PATs for rate-limit rotation (overrides environment)",
118 )
119
120
121def _exit_missing_token() -> None:
122 print("GitHub token required for API collection.", file=sys.stderr)
123 print(" Option A: copy .env.example to .env and set GITHUB_TOKEN_SE4DE", file=sys.stderr)
124 print(" Option B: $env:GITHUB_TOKEN_SE4DE = '<your-pat>' (PowerShell)", file=sys.stderr)
125 print(" Option C: pass --token <your-pat> on scripts that support it", file=sys.stderr)
126 sys.exit(1)
127 