PerturbReason/PerturbReason_dataset_code

1#!/bin/bash2#SBATCH --job-name=summer_04263#SBATCH --chdir=/scratch/group/PROJECT_NAME/OMICS_DATA/baseline_collections/summer_official4#SBATCH --output=/scratch/group/PROJECT_NAME/OMICS_DATA/baseline_collections/summer_official/logs/summer_0426_%A_%a.log5#SBATCH --partition=gpu6#SBATCH --nodes=17#SBATCH --ntasks-per-node=18#SBATCH --gres=gpu:a100:29#SBATCH --cpus-per-task=1610#SBATCH --mem=128G11#SBATCH --time=18:00:0012#SBATCH --account=ACCOUNT_ID13#SBATCH --array=0-914 15# ==============================================================================16# ENVIRONMENT SETUP17# ==============================================================================18module purge19module load CUDA/12.8.020 21# Cache directories22export CACHE_ROOT="/scratch/group/PROJECT_NAME/cache"23export HF_HOME="$CACHE_ROOT/hf"24export XDG_CACHE_HOME="$CACHE_ROOT/xdg_cache"25export VLLM_CONFIG_ROOT="$CACHE_ROOT/vllm_config"26export TRITON_CACHE_DIR="$CACHE_ROOT/triton"27export FLASHINFER_CACHE_DIR="$CACHE_ROOT/flashinfer"28 29mkdir -p "$HF_HOME" "$XDG_CACHE_HOME" "$VLLM_CONFIG_ROOT" "$TRITON_CACHE_DIR" "$FLASHINFER_CACHE_DIR"30 31# vLLM Optimizations32export VLLM_HOST_IP=$(hostname -I | awk '{print $1}')33export VLLM_USE_TORCH_COMPILE=034export VLLM_WORKER_MULTIPROC_METHOD=spawn35 36# Python Environment37PYTHON_BIN="/scratch/user/USER_NAME/State/state/.venv/bin/python"38export PATH="/scratch/user/USER_NAME/State/state/.venv/bin:$PATH"39 40# ==============================================================================41# PATHS42# ==============================================================================43KG_PATH="/scratch/group/PROJECT_NAME/OMICS_DATA/baseline_collections/kg"44DATASET_ROOT="/scratch/group/PROJECT_NAME/OMICS_DATA/baseline_collections/dataset_0426_final"45OUTPUT_ROOT="/scratch/group/PROJECT_NAME/OMICS_DATA/baseline_collections/summer_official"46 47# Find Model48MODEL_ROOT="/scratch/group/PROJECT_NAME/cache/hf/hub/models--Qwen--Qwen3-4B-Instruct-2507"49MODEL_PATH=$(find "$MODEL_ROOT/snapshots" -maxdepth 1 -mindepth 1 -type d | head -n 1)50 51# ==============================================================================52# JOB ARRAY MAPPING: 2 contexts × 5 splits = 10 jobs (0-9)53# Context 0: hidden_context  (jobs 0-4)54# Context 1: noisy_context   (jobs 5-9)55# Splits: id_test, cell_ood, pert_ood, double_ood, other_test56# ==============================================================================57CONTEXTS=(hidden_context noisy_context)58SPLITS=(id_test cell_ood pert_ood double_ood other_test)59 60CONTEXT_IDX=$(( SLURM_ARRAY_TASK_ID / 5 ))61SPLIT_IDX=$(( SLURM_ARRAY_TASK_ID % 5 ))62 63CONTEXT=${CONTEXTS[$CONTEXT_IDX]}64SPLIT=${SPLITS[$SPLIT_IDX]}65 66# Map context names to output directory names67declare -A CTX_TO_OUT68CTX_TO_OUT[hidden_context]=hidden_context_output_042669CTX_TO_OUT[noisy_context]=noisy_context_output_042670 71OUTPUT_NAME=${CTX_TO_OUT[$CONTEXT]}72 73INPUT_DIR="${DATASET_ROOT}/${CONTEXT}/${SPLIT}"74OUTPUT_DIR="${OUTPUT_ROOT}/${OUTPUT_NAME}/${SPLIT}"75 76# ==============================================================================77# EXECUTION78# ==============================================================================79echo "=========================================="80echo "SUMMER 3way Pipeline — dataset_0426_final"81echo "Context : $CONTEXT"82echo "Split   : $SPLIT  (array index $SLURM_ARRAY_TASK_ID)"83echo "Model   : $MODEL_PATH"84echo "Input   : $INPUT_DIR"85echo "Output  : $OUTPUT_DIR"86echo "=========================================="87 88mkdir -p "$OUTPUT_DIR"89mkdir -p /scratch/group/PROJECT_NAME/OMICS_DATA/baseline_collections/summer_official/logs90 91$PYTHON_BIN summer_vllm_v2.py \92    --kg_path "$KG_PATH" \93    --input_dir "$INPUT_DIR" \94    --output_dir "$OUTPUT_DIR" \95    --model_path "$MODEL_PATH" \96    --temperature 0.2 \97    --max_tokens 4096 \98    --tp_size 1 \99    --gpu_memory_utilization 0.9100 101echo "Finished: $CONTEXT / $SPLIT"