Cccccz/HY

1#!/usr/bin/env bash2set -euo pipefail3 4if [[ "$#" -ne 5 ]]; then5  echo "Usage: $0 EVAL_NAME CONFIDENCE_CHECKPOINT REPLACE_STEPS THRESHOLD_MODE THRESHOLD" >&26  echo "THRESHOLD_MODE must be global or per_step." >&27  exit 28fi9 10repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"11eval_name="$1"12confidence_checkpoint="$2"13replace_steps="$3"14threshold_mode="$4"15threshold="$5"16 17python_bin="${PREDICTOR_PYTHON_BIN:-/mnt/local_nvme/cz/envs/hy_worldplay_july20260720/bin/python}"18dataset_name="hyworldplay_predictor_vbench_val25_test50"19dataset_root="${PREDICTOR_HOLDOUT_NVME_ROOT:-${repo_root}/datasets/${dataset_name}}"20validation_root="${dataset_root}/validation"21eval_root="${dataset_root}/evaluations/${eval_name}"22log_dir="${eval_root}/logs"23base_model="${PREDICTOR_BASE_MODEL:-${repo_root}/outputs/hf_snapshot_repaired_9b49404}"24action_ckpt="${PREDICTOR_ACTION_CKPT:-/mnt/local_nvme/cz/checkpoints/hy_worldplay/huggingface/hub/models--tencent--HY-WorldPlay/snapshots/f4c29235647707b571479a69b569e4166f9f5bf8/ar_distilled_action_model/diffusion_pytorch_model.safetensors}"25weights="${repo_root}/checkpoints/predictor_v4_prefeature_blocks1-52_fp32master_8gpu_flr1e-4_blr1e-5_schedsteps2000_swarmup100_bwarmup100_swanlab/predictor_step_02000.safetensors"26 27if [[ ! "${eval_name}" =~ ^[A-Za-z0-9._-]+$ ]]; then28  echo "Invalid eval name: ${eval_name}" >&229  exit 230fi31if [[ "${replace_steps}" != "1,2" && "${replace_steps}" != "1,2,3" ]]; then32  echo "replace_steps must be 1,2 or 1,2,3" >&233  exit 234fi35if [[ "${threshold_mode}" != "global" && "${threshold_mode}" != "per_step" ]]; then36  echo "threshold mode must be global or per_step" >&237  exit 238fi39 40[[ -x "${python_bin}" ]]41[[ -d "${base_model}" ]]42[[ -f "${action_ckpt}" ]]43[[ -f "${weights}" ]]44[[ -f "${confidence_checkpoint}" ]]45[[ -f "${validation_root}/cases.jsonl" ]]46[[ $(find "${validation_root}/input_images" -type f -name '*.jpg' | wc -l) -eq 25 ]]47[[ $(find "${validation_root}/full" -type f -name '*.mp4' | wc -l) -eq 100 ]]48 49mkdir -p "${log_dir}"50export PYTHONPATH="${repo_root}"51export HY_PROFILE_TIMING=152 53confidence_args=(--confidence_checkpoint "${confidence_checkpoint}")54if [[ "${threshold_mode}" == "global" ]]; then55  confidence_args+=(--confidence_threshold "${threshold}")56else57  confidence_args+=(--confidence_thresholds "${threshold}")58fi59 60run_workers() {61  local phase="$1"62  shift63  local -a pids=()64  local worker_id pid failed=065  for worker_id in $(seq 0 7); do66    CUDA_VISIBLE_DEVICES="${worker_id}" "${python_bin}" -u "$@" \67      --worker_id "${worker_id}" --num_workers 8 \68      >"${log_dir}/${phase}_worker_${worker_id}.log" 2>&1 &69    pid="$!"70    pids+=("${pid}")71    echo "${pid}" >"${log_dir}/${phase}_worker_${worker_id}.pid"72  done73  for pid in "${pids[@]}"; do74    if ! wait "${pid}"; then failed=1; fi75  done76  if [[ "${failed}" -ne 0 ]]; then77    echo "${phase} worker failure; inspect ${log_dir}/${phase}_worker_*.log" >&278    return 179  fi80}81 82if [[ ! -f "${eval_root}/generation.completed" ]] \83  || [[ $(find "${eval_root}/predictor" -type f -name '*.mp4' 2>/dev/null | wc -l) -ne 100 ]]; then84  date -u +%FT%TZ >"${eval_root}/generation.started"85  run_workers generate \86    "${repo_root}/tools/generate_predictor_v4_train_case_eval.py" \87    --dataset_root "${validation_root}" \88    --output_root "${eval_root}" \89    --base_model "${base_model}" \90    --action_ckpt "${action_ckpt}" \91    --weights "${weights}" \92    --source_blocks 1,52 \93    --predictor_variant v4 \94    --previous_condition_mode same_step \95    --replace_steps "${replace_steps}" \96    --modes predictor \97    "${confidence_args[@]}"98  [[ $(find "${eval_root}/predictor" -type f -name '*.mp4' | wc -l) -eq 100 ]]99  date -u +%FT%TZ >"${eval_root}/generation.completed"100else101  echo "[skip] generation already complete: ${eval_name}"102fi103 104metric_records=0105if [[ -f "${eval_root}/metrics.json" ]]; then106  metric_records=$("${python_bin}" -c \107    'import json,sys; print(len(json.load(open(sys.argv[1]))["records"]))' \108    "${eval_root}/metrics.json")109fi110if [[ ! -f "${eval_root}/metrics.completed" || "${metric_records}" -ne 100 ]]; then111  date -u +%FT%TZ >"${eval_root}/metrics.started"112  run_workers metrics \113    "${repo_root}/tools/compute_predictor_v4_train_case_metrics.py" \114    --dataset_root "${validation_root}" \115    --output_root "${eval_root}" \116    --num_cases 25 \117    --reference_subdir full \118    --modes predictor \119    --metric_batch_size 4120  "${python_bin}" "${repo_root}/tools/merge_predictor_v4_train_case_metrics.py" \121    --output_root "${eval_root}" \122    --num_workers 8 \123    --num_cases 25 \124    --modes predictor \125    --replace_steps "${replace_steps}" \126    --reference_label "Full-DiT validation videos" \127    >"${log_dir}/metrics_merge.log" 2>&1128  [[ $("${python_bin}" -c \129    'import json,sys; print(len(json.load(open(sys.argv[1]))["records"]))' \130    "${eval_root}/metrics.json") -eq 100 ]]131  date -u +%FT%TZ >"${eval_root}/metrics.completed"132else133  echo "[skip] metrics already complete: ${eval_name}"134fi135 136date -u +%FT%TZ >"${eval_root}/sync.completed"137if [[ "${PREDICTOR_SKIP_SYNC:-1}" != "1" ]]; then138  bucket="${BUCKET:-s3-us-west2-default}"139  durable_eval_uri="s3://${bucket}/zoubin/cz/projects/HY-WorldPlay-DEV-Predictor/datasets/${dataset_name}/evaluations/${eval_name}"140  aws s3 sync "${eval_root}" "${durable_eval_uri}" --only-show-errors141fi142 143echo "Validation evaluation complete: ${eval_root}"144