altool/airflow

1#!/bin/bash2# ===========================================3# Airflow Configuration — Single Source of Truth4# ===========================================5# Override any value by setting the corresponding environment6# variable (e.g. as an HF Secret).7 8# ---- Directory Paths ----9export AIRFLOW_HOME="${AIRFLOW_HOME:-/opt/airflow}"10export DAGS_DIR="${AIRFLOW_HOME}/dags"11export LOGS_DIR="${AIRFLOW_HOME}/logs"12 13# ---- Database ----14# Default: local SQLite so the container always boots.15# For production, set AIRFLOW__DATABASE__SQL_ALCHEMY_CONN as an HF Secret16# pointing to your PostgreSQL instance.17export AIRFLOW__DATABASE__SQL_ALCHEMY_CONN="${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN:-sqlite:///${AIRFLOW_HOME}/airflow.db}"18 19# Pool tuning — keeps connections low for Supabase / free-tier DBs20export AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE="${AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_SIZE:-2}"21export AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW="${AIRFLOW__DATABASE__SQL_ALCHEMY_MAX_OVERFLOW:-2}"22export AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_RECYCLE="${AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_RECYCLE:-1800}"23export AIRFLOW__DATABASE__SQL_ALCHEMY_POOL_PRE_PING="True"24 25# ---- DAG Repository ----26export DAG_REPO_URL="${DAG_REPO_URL:-https://github.com/subhamgiri460/myworkflows.git}"27export DAG_REPO_BRANCH="${DAG_REPO_BRANCH:-main}"28# For private repos: set DAG_REPO_TOKEN to a GitHub PAT (fine-grained or classic)29# The token is injected into the URL at runtime — never stored in config.30export DAG_REPO_TOKEN="${DAG_REPO_TOKEN:-}"31 32# ---- Admin Credentials ----33# Set these as HF Secrets for production.34export AIRFLOW_ADMIN_USER="${AIRFLOW_ADMIN_USER:-admin}"35export AIRFLOW_ADMIN_PASSWORD="${AIRFLOW_ADMIN_PASSWORD:-admin}"36 37# ---- Internal Ports ----38export AIRFLOW_WEB_PORT="${AIRFLOW_WEB_PORT:-8080}"39export REFRESH_SERVICE_PORT="${REFRESH_SERVICE_PORT:-5000}"40export NGINX_PORT="${NGINX_PORT:-7860}"41 42# ---- Core ----43export AIRFLOW__CORE__LOAD_EXAMPLES="${AIRFLOW__CORE__LOAD_EXAMPLES:-False}"44export AIRFLOW__CORE__EXECUTOR="${AIRFLOW__CORE__EXECUTOR:-LocalExecutor}"45export AIRFLOW__CORE__PARALLELISM="${AIRFLOW__CORE__PARALLELISM:-2}"46export AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG="${AIRFLOW__CORE__MAX_ACTIVE_TASKS_PER_DAG:-2}"47export AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION="${AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION:-True}"48 49# Auto-generate Fernet key if not set (required for connection encryption)50if [ -z "${AIRFLOW__CORE__FERNET_KEY:-}" ]; then51    export AIRFLOW__CORE__FERNET_KEY=$(python3 -c "from cryptography.fernet import Fernet; print(Fernet.generate_key().decode())" 2>/dev/null || echo "")52fi53 54# ---- Webserver ----55export AIRFLOW__WEBSERVER__WEB_SERVER_PORT="${AIRFLOW_WEB_PORT}"56export AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT="${AIRFLOW__WEBSERVER__WEB_SERVER_MASTER_TIMEOUT:-300}"57export AIRFLOW__WEBSERVER__WORKER_CLASS="${AIRFLOW__WEBSERVER__WORKER_CLASS:-gevent}"58export AIRFLOW__WEBSERVER__SECRET_KEY="${AIRFLOW__WEBSERVER__SECRET_KEY:-$(python3 -c 'import secrets; print(secrets.token_hex(32))' 2>/dev/null)}"59export AIRFLOW__WEBSERVER__WORKERS="${AIRFLOW__WEBSERVER__WORKERS:-1}"60export AIRFLOW__WEBSERVER__EXPOSE_CONFIG="${AIRFLOW__WEBSERVER__EXPOSE_CONFIG:-False}"61 62# ---- Scheduler ----63export AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL="${AIRFLOW__SCHEDULER__DAG_DIR_LIST_INTERVAL:-30}"64 65# ---- API ----66export AIRFLOW__API__AUTH_BACKENDS="${AIRFLOW__API__AUTH_BACKENDS:-airflow.api.auth.backend.session}"67 68# ---- Logging ----69export AIRFLOW__LOGGING__LOGGING_LEVEL="${AIRFLOW__LOGGING__LOGGING_LEVEL:-INFO}"70 71# ---- Validation ----72validate_config() {73    local warn=074    if [[ "${AIRFLOW__DATABASE__SQL_ALCHEMY_CONN}" == sqlite* ]]; then75        echo "[WARN] Using SQLite — not suitable for production. Set AIRFLOW__DATABASE__SQL_ALCHEMY_CONN as an HF Secret."76        # SQLite requires SequentialExecutor77        export AIRFLOW__CORE__EXECUTOR="SequentialExecutor"78        warn=179    fi80    if [ "${AIRFLOW_ADMIN_PASSWORD}" = "admin" ]; then81        echo "[WARN] Using default admin password. Set AIRFLOW_ADMIN_PASSWORD as an HF Secret."82        warn=183    fi84    if [ "${warn}" -eq 0 ]; then85        echo "[OK] Configuration validated."86    fi87}