SHUBHAMOS/meta-pytorch-hackathon

1#!/bin/bash2# E2E Runner Script for SHUBHAMOS (Phase 8 Validation)3 4set -e5 6# Constraints: Read HF_TOKEN or NVIDIA_API_KEY from environment7if [ -z "$HF_TOKEN" ] && [ -z "$NVIDIA_API_KEY" ]; then8    echo "❌ ERROR: Neither HF_TOKEN nor NVIDIA_API_KEY environment variable is set."9    echo "Usage: export HF_TOKEN='...' or export NVIDIA_API_KEY='...' && ./tests/e2e_runner.sh"10    exit 111fi12 13mkdir -p logs14mkdir -p tests15 16SUMMARY_FILE="tests/summary_report.txt"17echo "SHUBHAMOS E2E Validation Report" > $SUMMARY_FILE18echo "===============================" >> $SUMMARY_FILE19date >> $SUMMARY_FILE20echo "" >> $SUMMARY_FILE21 22# Function to run, validate, and extract metrics for a task23run_and_validate() {24    local task=$125    local log="logs/${task}.log"26    27    echo "Running E2E for TASK=${task}... (This may take several minutes)"28    # We allow the python script to fail naturally without breaking the wrapper29    python3 -u inference.py --task $task > $log 2>&1 || true30 31    echo "Analyzing $log..."32    33    # 1. Check for crashes (Traceback)34    if grep -q "Traceback (most recent call last):" "$log"; then35        echo "❌ FAIL: Unhandled Traceback Exception found in log!"36        echo "Task: $task | Status: FAIL | Reason: Runtime Crash" >> $SUMMARY_FILE37        return 138    fi39    40    # 2. Check for FINAL SCORE signature41    if ! grep -q "FINAL SCORE ($task):" "$log"; then42        echo "❌ FAIL: 'FINAL SCORE ($task):' signature missing from log!"43        echo "Task: $task | Status: FAIL | Reason: Missing Score Output" >> $SUMMARY_FILE44        return 145    fi46    47    # 3. Extract final score and validate boundaries (0.0 to 1.0)48    local score=$(grep "FINAL SCORE ($task):" "$log" | awk -F': ' '{print $2}')49    local is_valid_score=$(echo "$score" | awk '{if ($1 >= 0.0 && $1 <= 1.0) print "yes"; else print "no"}')50 51    if [ "$is_valid_score" != "yes" ]; then52        echo "❌ FAIL: Final Score ($score) is outside bound [0.0 - 1.0]!"53        echo "Task: $task | Score: $score | Status: FAIL | Reason: Out of Bounds" >> $SUMMARY_FILE54        return 155    fi56 57    # 4. Fallback Metrics extraction58    local total_steps=$(grep -c "^\[STEP " "$log" || echo "0")59    local fallbacks=$(grep -c "^Fallback: YES" "$log" || echo "0")60    61    local fallback_rate=$(awk -v f="$fallbacks" -v t="$total_steps" 'BEGIN { if (t > 0) printf "%.2f", (f / t) * 100; else print "0.00" }')62 63    echo "✅ PASS: $task executed successfully."64    echo "  - Total Steps: $total_steps"65    echo "  - Fallbacks:   $fallbacks ($fallback_rate%)"66    echo "  - Final Score: $score"67    68    echo "Task: $task | Score: $score | Fallback Rate: $fallback_rate% | Status: PASS" >> $SUMMARY_FILE69    return 070}71 72# Run tasks73FAILED=074 75run_and_validate "easy"76if [ $? -ne 0 ]; then FAILED=1; fi77 78echo "-----------------------------------"79 80run_and_validate "medium"81if [ $? -ne 0 ]; then FAILED=1; fi82 83echo "==============================="84cat $SUMMARY_FILE85 86if [ $FAILED -eq 1 ]; then87    echo "❌ E2E Validation Failed."88    exit 189else90    echo "✅ All E2E checks passed perfectly."91    exit 092fi93