SignerX/SignVerse-2M
SignVerse-2M SignVerse-2M: A Two-Million-Clip Pose-Native Universe of 55+ Sign Languages Links: [Paper] | [Data Files] | [Project Page] SignVerse-2M is a large-scale multilingual pose-native dataset for sign language research. The dataset reorganizes publicly available sign language videos into a unified DWPose-based representation and releases the result as approximately 2 million clips from 39,196 videos covering 55+ sign languages. Rather than… See the full description on the dataset page: https://huggingface.co/datasets/SignerX/SignVerse-2M.
101.9k
1#!/usr/bin/env bash2#SBATCH --job-name=upload3#SBATCH --nodes=14#SBATCH --ntasks=15#SBATCH --cpus-per-task=26#SBATCH --mem=8G7#SBATCH --time=24:00:008#SBATCH --output=%x_%A_%a.out9#SBATCH --error=%x_%A_%a.err10 11set -euo pipefail12 13ROOT_DIR="${ROOT_DIR:-/home/sf895/SignVerse-2M}"14RUNTIME_ROOT="${RUNTIME_ROOT:-/home/sf895/SignVerse-2M-runtime}"15CONDA_SH="${CONDA_SH:-/home/sf895/miniconda3/etc/profile.d/conda.sh}"16CONDA_ENV="${CONDA_ENV:-signx2}"17PIPELINE03="${PIPELINE03:-$ROOT_DIR/scripts/pipeline03_upload_to_huggingface.py}"18DATASET_DIR="${DATASET_DIR:-$RUNTIME_ROOT/dataset}"19SCRATCH_DATASET_DIR="${SCRATCH_DATASET_DIR:-/scratch/$USER/SignVerse-2M-runtime/dataset}"20RAW_VIDEO_DIR="${RAW_VIDEO_DIR:-$RUNTIME_ROOT/raw_video}"21SCRATCH_RAW_VIDEO_DIR="${SCRATCH_RAW_VIDEO_DIR:-/scratch/$USER/SignVerse-2M-runtime/raw_video}"22RAW_CAPTION_DIR="${RAW_CAPTION_DIR:-$RUNTIME_ROOT/raw_caption}"23RAW_METADATA_DIR="${RAW_METADATA_DIR:-$RUNTIME_ROOT/raw_metadata}"24ARCHIVE_DIR="${ARCHIVE_DIR:-$ROOT_DIR/archives}"25PROGRESS_JSON="${PROGRESS_JSON:-$RUNTIME_ROOT/archive_upload_progress.json}"26STATUS_JOURNAL_PATH="${STATUS_JOURNAL_PATH:-$RUNTIME_ROOT/upload_status_journal.jsonl}"27STATS_NPZ="${STATS_NPZ:-$RUNTIME_ROOT/stats.npz}"28REPO_ID="${REPO_ID:-SignerX/SignVerse-2M}"29REPO_REVISION="${REPO_REVISION:-dev}"30TARGET_BYTES="${TARGET_BYTES:-10737418240}"31TARGET_FOLDERS="${TARGET_FOLDERS:-40}"32PARALLEL_SHARDS="${PARALLEL_SHARDS:-1}"33START_STAGGER_MIN="${START_STAGGER_MIN:-1}"34START_STAGGER_MAX="${START_STAGGER_MAX:-3}"35ALLOW_SMALL_FINAL_BATCH="${ALLOW_SMALL_FINAL_BATCH:-0}"36REQUIRE_TARGET_BYTES="${REQUIRE_TARGET_BYTES:-1}"37DRY_RUN_UPLOAD="${DRY_RUN_UPLOAD:-0}"38UPLOAD_MODE="${UPLOAD_MODE:-api}"39 40if [[ ! -f "$CONDA_SH" ]]; then41 echo "Missing conda init script: $CONDA_SH" >&242 exit 143fi44 45if [[ -z "${SLURM_ARRAY_TASK_ID:-}" ]]; then46 echo "SLURM_ARRAY_TASK_ID is required." >&247 exit 148fi49 50if (( PARALLEL_SHARDS < 1 )); then51 echo "PARALLEL_SHARDS must be >= 1" >&252 exit 153fi54 55if (( SLURM_ARRAY_TASK_ID < 0 || SLURM_ARRAY_TASK_ID >= PARALLEL_SHARDS )); then56 echo "SLURM_ARRAY_TASK_ID=$SLURM_ARRAY_TASK_ID is out of range for PARALLEL_SHARDS=$PARALLEL_SHARDS" >&257 exit 158fi59 60echo "[$(date '+%F %T')] upload shard=${SLURM_ARRAY_TASK_ID}/${PARALLEL_SHARDS} host=$(hostname)"61 62# shellcheck disable=SC109063source "$CONDA_SH"64 65cmd=(python -u "$PIPELINE03"66 --dataset-dir "$DATASET_DIR"67 --scratch-dataset-dir "$SCRATCH_DATASET_DIR"68 --raw-video-dir "$RAW_VIDEO_DIR"69 --scratch-raw-video-dir "$SCRATCH_RAW_VIDEO_DIR"70 --raw-caption-dir "$RAW_CAPTION_DIR"71 --raw-metadata-dir "$RAW_METADATA_DIR"72 --archive-dir "$ARCHIVE_DIR"73 --progress-path "$PROGRESS_JSON"74 --stats-npz "$STATS_NPZ"75 --status-journal-path "$STATUS_JOURNAL_PATH"76 --repo-id "$REPO_ID"77 --repo-revision "$REPO_REVISION"78 --target-bytes "$TARGET_BYTES"79 --target-folders "$TARGET_FOLDERS"80 --parallel-shards "$PARALLEL_SHARDS"81 --shard-index "$SLURM_ARRAY_TASK_ID"82 --start-stagger-min "$START_STAGGER_MIN"83 --start-stagger-max "$START_STAGGER_MAX"84 --upload-mode "$UPLOAD_MODE"85)86 87if [[ "$ALLOW_SMALL_FINAL_BATCH" == "1" ]]; then88 cmd+=(--allow-small-final-batch)89fi90if [[ "$REQUIRE_TARGET_BYTES" != "1" ]]; then91 cmd+=(--allow-small-final-batch)92fi93if [[ "$DRY_RUN_UPLOAD" == "1" ]]; then94 cmd+=(--dry-run)95fi96 97cmd+=(--skip-stats-write)98 99CONDA_NO_PLUGINS=true conda run -n "$CONDA_ENV" "${cmd[@]}"100