hftestx/67b8b004
0
1#!/bin/bash2 3set -euo pipefail # Strict error handling4 5# Set timezone to UTC for consistent timestamp handling across environments6export TZ=UTC7 8# Default configuration file path9DEFAULT_CONFIG_FILE="${CONFIG_FILE:-/home/user/config/persistence.conf}"10 11# Logging functions12log() {13 local level="$1"14 shift15 # Ensure log directory exists16 mkdir -p "$(dirname "${LOG_FILE:-/home/user/log/persistence.log}")"17 echo "[$(date '+%Y-%m-%d %H:%M:%S')] [$level] $*" | tee -a "${LOG_FILE:-/home/user/log/persistence.log}"18}19 20log_info() { log "INFO" "$@"; }21log_warn() { log "WARN" "$@"; }22log_error() { log "ERROR" "$@"; }23 24# Load configuration file25load_configuration() {26 local config_file="${1:-$DEFAULT_CONFIG_FILE}"27 28 if [[ ! -f "$config_file" ]]; then29 log_warn "Configuration file does not exist: $config_file, using default configuration"30 return 031 fi32 33 log_info "Loading configuration file: $config_file"34 35 # Read shell variable format configuration file36 source "$config_file"37}38 39# Set default configuration40set_default_configuration() {41 # Core configuration42 export HF_TOKEN="${HF_TOKEN:-}"43 export DATASET_ID="${DATASET_ID:-}"44 export ARCHIVE_PATHS="${ARCHIVE_PATHS:-}"45 export RESTORE_PATH="${RESTORE_PATH:-./}"46 47 # Sync configuration48 export SYNC_INTERVAL="${SYNC_INTERVAL:-}" # 2 hours49 export MAX_ARCHIVES="${MAX_ARCHIVES:-}"50 export COMPRESSION_LEVEL="${COMPRESSION_LEVEL:-}"51 export INITIAL_BACKUP_DELAY="${INITIAL_BACKUP_DELAY:-}" # 5 minutes52 53 # File configuration54 export ARCHIVE_PREFIX="${ARCHIVE_PREFIX:-}"55 export ARCHIVE_EXTENSION="${ARCHIVE_EXTENSION:-}"56 export EXCLUDE_PATTERNS="${EXCLUDE_PATTERNS:-}"57 58 # Application configuration59 export APP_COMMAND="${APP_COMMAND:-}"60 export ENABLE_AUTO_RESTORE="${ENABLE_AUTO_RESTORE:-}"61 export ENABLE_AUTO_SYNC="${ENABLE_AUTO_SYNC:-}"62 63 # Synchronous restore configuration64 export FORCE_SYNC_RESTORE="${FORCE_SYNC_RESTORE:-}"65 export ENABLE_INTEGRITY_CHECK="${ENABLE_INTEGRITY_CHECK:-}"66 67 # Logging configuration68 export LOG_FILE="${LOG_FILE:-}"69 export LOG_LEVEL="${LOG_LEVEL:-}"70}71 72# Validate required environment variables73validate_configuration() {74 local errors=075 76 if [[ -z "$HF_TOKEN" ]]; then77 log_error "Missing required environment variable: HF_TOKEN"78 ((errors++))79 fi80 81 if [[ -z "$DATASET_ID" ]]; then82 log_error "Missing required environment variable: DATASET_ID"83 ((errors++))84 fi85 86 if [[ $errors -gt 0 ]]; then87 log_error "Configuration validation failed, starting application in non-persistent mode"88 return 189 fi90 91 # Set Hugging Face authentication92 export HUGGING_FACE_HUB_TOKEN="$HF_TOKEN"93 94 log_info "Configuration validation successful"95 return 096}97 98# Create archive file99create_archive() {100 local timestamp=$(date +%Y%m%d_%H%M%S)101 local archive_file="${ARCHIVE_PREFIX}_${timestamp}.${ARCHIVE_EXTENSION}"102 # Use user-owned directory instead of /tmp103 local temp_dir="/home/user/temp"104 mkdir -p "$temp_dir"105 local temp_archive="$temp_dir/${archive_file}"106 107 log_info "Starting archive creation: $archive_file" >&2108 109 # Build exclude arguments - only exclude files that would negatively impact HuggingFace datasets backup110 local exclude_args=""111 local default_excludes="__pycache__,*.tmp,*/temp,*/cache,*/.cache,*/log,*/logs"112 local combined_patterns="${EXCLUDE_PATTERNS:-},${default_excludes}"113 114 if [[ -n "$combined_patterns" ]]; then115 IFS=',' read -ra patterns <<< "$combined_patterns"116 for pattern in "${patterns[@]}"; do117 pattern="${pattern// /}" # Remove spaces118 if [[ -n "$pattern" ]]; then119 exclude_args+=" --exclude='${pattern}'"120 fi121 done122 fi123 124 # Add tar options to handle file changes and permission issues125 # Use UTC timezone for consistent timestamp handling126 local tar_options="--ignore-failed-read --warning=no-file-changed --warning=no-file-removed --mtime='@$(date +%s)'"127 128 # Create archive129 local archive_paths_array130 IFS=',' read -ra archive_paths_array <<< "$ARCHIVE_PATHS"131 132 local tar_cmd="tar -czf '$temp_archive' $tar_options $exclude_args"133 local valid_paths=()134 135 for path in "${archive_paths_array[@]}"; do136 path="${path// /}" # Remove spaces137 if [[ -e "$path" ]]; then138 # Check if directory is empty139 if [[ -d "$path" ]] && [[ -z "$(ls -A "$path" 2>/dev/null)" ]]; then140 log_warn "Directory is empty, creating placeholder file: $path" >&2141 echo "# Placeholder file for persistence backup" > "$path/.persistence_placeholder"142 fi143 tar_cmd+=" '$path'"144 valid_paths+=("$path")145 else146 log_warn "Archive path does not exist, skipping: $path" >&2147 fi148 done149 150 # Check if there are valid paths151 if [[ ${#valid_paths[@]} -eq 0 ]]; then152 log_error "No valid archive paths found" >&2153 return 1154 fi155 156 log_info "Executing archive command: $tar_cmd" >&2157 158 # Execute tar command and capture exit code159 local tar_exit_code=0160 if eval "$tar_cmd" >&2; then161 tar_exit_code=$?162 else163 tar_exit_code=$?164 fi165 166 # Check if archive was created successfully (tar exit code 0 or 1 is acceptable)167 # Exit code 1 means some files changed during archiving, which is normal168 if [[ $tar_exit_code -eq 0 || $tar_exit_code -eq 1 ]] && [[ -f "$temp_archive" ]]; then169 log_info "Archive file created successfully: $temp_archive" >&2170 if [[ $tar_exit_code -eq 1 ]]; then171 log_warn "Some files changed during archiving (this is normal for active applications)" >&2172 fi173 echo "$temp_archive"174 return 0175 else176 log_error "Archive file creation failed with exit code: $tar_exit_code" >&2177 return 1178 fi179}180 181# Call Python upload handler with pre-upload cleanup182run_upload_handler() {183 local archive_file="$1"184 local filename="$2"185 local dataset_id="$3"186 local backup_prefix="$4"187 local backup_extension="$5"188 local max_backups="$6"189 local token="$7"190 191 # Get script directory for relative imports192 local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"193 194 # Call the standalone Python module (now with pre-upload cleanup logic)195 python3 "${script_dir}/hf_persistence.py" upload \196 --token "$token" \197 --dataset-id "$dataset_id" \198 --archive-file "$archive_file" \199 --filename "$filename" \200 --archive-prefix "$backup_prefix" \201 --archive-extension "$backup_extension" \202 --max-archives "$max_backups"203}204 205# Upload archive to Hugging Face206upload_archive() {207 local archive_file="$1"208 local filename=$(basename "$archive_file")209 210 log_info "Starting archive upload: $filename"211 212 # Call embedded Python handler213 if run_upload_handler "$archive_file" "$filename" "$DATASET_ID" "$ARCHIVE_PREFIX" "$ARCHIVE_EXTENSION" "$MAX_ARCHIVES" "$HF_TOKEN"; then214 log_info "Archive upload completed"215 return 0216 else217 log_error "Archive upload failed"218 return 1219 fi220}221 222# Perform one archive operation223perform_archive() {224 log_info "Starting archive operation"225 226 local archive_file227 if archive_file=$(create_archive); then228 # Check if in test mode (HF_TOKEN is test_token)229 if [[ "$HF_TOKEN" == "test_token" ]]; then230 log_info "Test mode: Archive created successfully, skipping upload"231 log_info "Archive file: $archive_file"232 ls -la "$archive_file"233 log_info "Test mode: Keeping archive file for inspection"234 else235 if upload_archive "$archive_file"; then236 log_info "Archive operation completed successfully"237 else238 log_error "Archive upload failed"239 fi240 # Clean up temporary files241 rm -f "$archive_file"242 fi243 else244 log_error "Archive creation failed"245 return 1246 fi247}248 249# Sync daemon250sync_daemon() {251 log_info "Starting sync daemon, interval: ${SYNC_INTERVAL} seconds"252 253 # Initial delay to allow application to fully start254 local initial_delay="${INITIAL_BACKUP_DELAY:-300}"255 log_info "Waiting ${initial_delay} seconds for application to fully initialize before first backup"256 sleep "$initial_delay"257 258 while true; do259 perform_archive260 261 log_info "Next sync will execute in ${SYNC_INTERVAL} seconds"262 sleep "$SYNC_INTERVAL"263 done264}265 266# Call Python archive lister267run_archive_lister() {268 local dataset_id="$1"269 local backup_prefix="$2"270 local backup_extension="$3"271 local token="$4"272 273 # Get script directory for relative imports274 local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"275 276 # Call the standalone Python module277 python3 "${script_dir}/hf_persistence.py" list \278 --token "$token" \279 --dataset-id "$dataset_id" \280 --archive-prefix "$backup_prefix" \281 --archive-extension "$backup_extension"282}283 284# List available archives285list_archives() {286 log_info "Getting available archive list"287 288 # Check if in test mode (HF_TOKEN is test_token)289 if [[ "$HF_TOKEN" == "test_token" ]]; then290 log_info "Test mode: Simulating empty archive list"291 echo "No archive files found"292 return 0293 fi294 295 # Call embedded Python handler296 run_archive_lister "$DATASET_ID" "$ARCHIVE_PREFIX" "$ARCHIVE_EXTENSION" "$HF_TOKEN"297}298 299# Call Python download handler300run_download_handler() {301 local backup_name="$1"302 local dataset_id="$2"303 local restore_path="$3"304 local token="$4"305 306 # Get script directory for relative imports307 local script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"308 309 # Call the standalone Python module310 python3 "${script_dir}/hf_persistence.py" restore \311 --token "$token" \312 --dataset-id "$dataset_id" \313 --archive-name "$backup_name" \314 --restore-path "$restore_path"315}316 317# Verify data integrity after restoration318verify_data_integrity() {319 # Check if integrity verification is enabled320 if [[ "$ENABLE_INTEGRITY_CHECK" != "true" ]]; then321 log_info "Data integrity verification is disabled, skipping"322 return 0323 fi324 325 local verification_failed=0326 327 log_info "Starting data integrity verification"328 329 # Check if critical directories exist and are accessible330 IFS=',' read -ra archive_paths_array <<< "$ARCHIVE_PATHS"331 for path in "${archive_paths_array[@]}"; do332 path="${path// /}" # Remove spaces333 if [[ -n "$path" ]]; then334 if [[ -e "$path" ]]; then335 log_info "✓ Verified path exists: $path"336 # Check if directory is readable337 if [[ -d "$path" ]] && [[ ! -r "$path" ]]; then338 log_error "✗ Directory exists but is not readable: $path"339 ((verification_failed++))340 fi341 # Check if directory is writable (for future operations)342 if [[ -d "$path" ]] && [[ ! -w "$path" ]]; then343 log_error "✗ Directory exists but is not writable: $path"344 ((verification_failed++))345 fi346 else347 log_warn "⚠ Path does not exist after restoration: $path"348 # This might be acceptable for first run349 fi350 fi351 done352 353 # Additional integrity checks can be added here354 # For example, checking for specific configuration files355 356 if [[ $verification_failed -gt 0 ]]; then357 log_error "Data integrity verification failed with $verification_failed errors"358 return 1359 else360 log_info "✓ Data integrity verification passed"361 return 0362 fi363}364 365# Restore specified archive with integrity verification366restore_archive() {367 local archive_name="${1:-latest}"368 local force_restore="${2:-false}"369 370 log_info "Starting synchronous archive restoration: $archive_name"371 372 # If latest, get the latest archive name first373 if [[ "$archive_name" == "latest" ]]; then374 local archive_list_output375 if archive_list_output=$(list_archives 2>&1); then376 archive_name=$(echo "$archive_list_output" | grep "LATEST_BACKUP:" | cut -d: -f2)377 if [[ -z "$archive_name" ]]; then378 log_info "No archive files found, this appears to be the first run"379 if [[ "$force_restore" == "true" ]]; then380 log_info "Force restore requested but no archives available - this is normal for first run"381 log_info "Initializing fresh environment for first-time startup"382 383 # Create necessary directory structure for first run384 IFS=',' read -ra archive_paths_array <<< "$ARCHIVE_PATHS"385 for path in "${archive_paths_array[@]}"; do386 path="${path// /}" # Remove spaces387 if [[ -n "$path" ]] && [[ ! -e "$path" ]]; then388 log_info "Creating directory for first run: $path"389 mkdir -p "$path" || log_warn "Failed to create directory: $path"390 fi391 done392 393 log_info "✓ First-time environment initialization completed"394 return 0395 else396 log_info "Continuing with fresh start (no archives available)"397 return 0398 fi399 fi400 else401 # Check if output contains "No archive files found"402 if echo "$archive_list_output" | grep -q "No archive files found"; then403 log_info "No archive files found, this appears to be the first run"404 if [[ "$force_restore" == "true" ]]; then405 log_info "Force restore requested but no archives available - this is normal for first run"406 log_info "Initializing fresh environment for first-time startup"407 408 # Create necessary directory structure for first run409 IFS=',' read -ra archive_paths_array <<< "$ARCHIVE_PATHS"410 for path in "${archive_paths_array[@]}"; do411 path="${path// /}" # Remove spaces412 if [[ -n "$path" ]] && [[ ! -e "$path" ]]; then413 log_info "Creating directory for first run: $path"414 mkdir -p "$path" || log_warn "Failed to create directory: $path"415 fi416 done417 418 log_info "✓ First-time environment initialization completed"419 return 0420 else421 log_info "Continuing with fresh start (no archives available)"422 return 0423 fi424 else425 log_error "Failed to get archive list: $archive_list_output"426 return 1427 fi428 fi429 fi430 431 log_info "Restoring archive file: $archive_name"432 433 # Call embedded Python handler434 if run_download_handler "$archive_name" "$DATASET_ID" "$RESTORE_PATH" "$HF_TOKEN"; then435 log_info "Archive extraction completed, verifying data integrity..."436 437 # Verify data integrity after restoration438 if verify_data_integrity; then439 log_info "✓ Archive restoration completed successfully with integrity verification"440 return 0441 else442 log_error "✗ Data integrity verification failed after restoration"443 return 1444 fi445 else446 log_error "✗ Archive restoration failed during extraction"447 return 1448 fi449}450 451# Main program entry452main() {453 local command="start"454 local config_file="$DEFAULT_CONFIG_FILE"455 local verbose=false456 local no_restore=false457 local no_sync=false458 local restore_target=""459 460 # Parse command line arguments461 while [[ $# -gt 0 ]]; do462 case $1 in463 -c|--config)464 config_file="$2"465 shift 2466 ;;467 -v|--verbose)468 verbose=true469 shift470 ;;471 --no-restore)472 no_restore=true473 shift474 ;;475 --no-sync)476 no_sync=true477 shift478 ;;479 archive|restore|restore-sync|list|daemon|start)480 command="$1"481 shift482 ;;483 *)484 if [[ ("$command" == "restore" || "$command" == "restore-sync") && -z "$restore_target" ]]; then485 # Archive name parameter for restore and restore-sync commands486 restore_target="$1"487 shift488 else489 log_error "Unknown parameter: $1"490 exit 1491 fi492 ;;493 esac494 done495 496 # Load configuration497 load_configuration "$config_file"498 set_default_configuration499 500 # Set log level501 if [[ "$verbose" == "true" ]]; then502 export LOG_LEVEL="DEBUG"503 fi504 505 log_info "=== Data Persistence Single File Script Startup ==="506 log_info "Version: 4.0"507 log_info "Command: $command"508 log_info "Configuration file: $config_file"509 510 # Execute corresponding operation based on command511 case $command in512 archive)513 if validate_configuration; then514 perform_archive515 else516 exit 1517 fi518 ;;519 restore)520 if validate_configuration; then521 restore_archive "${restore_target:-latest}"522 else523 exit 1524 fi525 ;;526 restore-sync)527 # Synchronous restore with mandatory integrity verification528 if validate_configuration; then529 log_info "=== SYNCHRONOUS RESTORE MODE ==="530 log_info "This is a blocking operation that must complete successfully"531 532 if restore_archive "${restore_target:-latest}" "true"; then533 log_info "✓ Synchronous restore completed successfully"534 exit 0535 else536 log_error "✗ Synchronous restore failed"537 log_error "Operation aborted to prevent data inconsistency"538 exit 1539 fi540 else541 log_error "Configuration validation failed"542 exit 1543 fi544 ;;545 list)546 if validate_configuration; then547 list_archives548 else549 exit 1550 fi551 ;;552 daemon)553 if validate_configuration; then554 sync_daemon555 else556 exit 1557 fi558 ;;559 start)560 # Application startup mode with synchronous data restoration561 if validate_configuration; then562 # Synchronous auto restore - behavior depends on FORCE_SYNC_RESTORE563 if [[ "$ENABLE_AUTO_RESTORE" == "true" && "$no_restore" == "false" ]]; then564 log_info "=== SYNCHRONOUS DATA RESTORATION PHASE ==="565 log_info "Performing synchronous auto restore - this is a blocking operation"566 log_info "Force sync restore: $FORCE_SYNC_RESTORE"567 log_info "Integrity check: $ENABLE_INTEGRITY_CHECK"568 569 if restore_archive "latest"; then570 log_info "✓ Synchronous data restoration completed successfully"571 log_info "All dependent services can now start safely"572 else573 if [[ "$FORCE_SYNC_RESTORE" == "true" ]]; then574 log_error "✗ Synchronous data restoration failed"575 log_error "FORCE_SYNC_RESTORE=true: Service startup will be aborted to prevent data inconsistency"576 log_error "Please check the logs and fix any issues before restarting"577 exit 1578 else579 log_warn "✗ Synchronous data restoration failed"580 log_warn "FORCE_SYNC_RESTORE=false: Continuing with service startup (legacy behavior)"581 log_warn "This may result in data inconsistency"582 fi583 fi584 585 log_info "=== DATA RESTORATION PHASE COMPLETED ==="586 else587 log_info "Auto restore is disabled, skipping data restoration"588 fi589 590 # Start sync daemon only after successful restoration591 if [[ "$ENABLE_AUTO_SYNC" == "true" && "$no_sync" == "false" ]]; then592 log_info "Starting sync daemon (data restoration completed)"593 sync_daemon &594 sync_pid=$!595 log_info "Sync daemon PID: $sync_pid"596 fi597 else598 log_warn "Configuration validation failed, starting application in non-persistent mode"599 log_warn "No data restoration will be performed"600 fi601 602 # Start main application only after all restoration is complete603 log_info "=== STARTING MAIN APPLICATION ==="604 log_info "All data restoration and verification completed"605 log_info "Starting main application: $APP_COMMAND"606 exec $APP_COMMAND607 ;;608 *)609 log_error "Unknown command: $command"610 exit 1611 ;;612 esac613}614 615# Script entry point616if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then617 main "$@"618fi