mm-eval/VSI-Bench

1{2  "name": "VSI-Bench",3  "release_date": "2024-12-18",4  "subsets": {5    "mc": {6      "language": [7        "en"8      ],9      "modalities": [10        "single_video_start"11      ],12      "task_type": "multiple_choice_qa",13      "score_pipeline": [14        "exact-match",15        "rule-match"16      ],17      "score_protocol": {18        "reference": "lmms-eval@lmms_eval/tasks/vsibench/utils.py:87-89,115-121 — MCA question types graded by fuzzy_matching (first token, strip trailing period) + case-insensitive exact match against the ground-truth letter; no LLM.",19        "note": "The org ships ONLY the multiple-choice question types (mc config, 2490 rows; confirmed in rows file). The official benchmark also contains NA (numerical) types graded with Mean Relative Accuracy MRA:.5:.95:.05 (utils.py:95-100,29-31), and the official headline 'overall' averages 8 per-question-type scores after averaging object_rel_direction easy/medium/hard (utils.py:158-170) — neither is reproducible from this copy or from a per-sample mean. Rows carry extra 'question_type' with VSI type names (object_rel_direction_hard, ...) that are not scorer vocabulary."20      },21      "prompt_template": "<video>These are frames of a video.\n{{ question }}\nOptions:\n{% for o in options %}{{ o }}\n{% endfor %}Answer with the option's letter from the given choices directly.",22      "prompt_template_source": {23        "origin": "official",24        "reference": "https://github.com/EvolvingLMMs-Lab/lmms-eval/blob/main/lmms_eval/tasks/vsibench/utils.py (vsibench_doc_to_text — canonical MCA prompt) + _default_template_yaml",25        "notes": "Tier 4: lmms-eval VSI-Bench canonical MCA prompt for the MC question types. SAT paper (arXiv 2412.07755, Appendix A.3, p.20) restricts VSI-Bench evaluation to the MC split — numerical splits (object_counting, object_size_estimation, room_size_estimation, object_abs_distance) are excluded. Each MC row carries the source `options` list verbatim from nyu-visionx/VSI-Bench under `choices`."26      },27      "video_storage": {28        "format": "files",29        "media_root": "videos",30        "notes": "512 source videos (243 referenced by the MC split): arkitscenes (121), scannet (79), scannetpp (43). Videos are mirrored from lccshunli/VSIBench at 5.7 GB total; nyu-visionx/VSI-Bench provides only QA metadata. Each scene_name resolves to videos/<dataset_source>/<scene_name>.mp4. Frame sampling is the inference backend's responsibility."31      },32      "mapping_from_source": {33        "media": {34          "from": "video_path",35          "type": "list",36          "min_items": 1,37          "max_items": 138        },39        "id": {40          "from": "id"41        },42        "question": {43          "from": "question"44        },45        "answer": {46          "from": "ground_truth"47        },48        "choices": {49          "from": "options"50        },51        "extra": {52          "scene_name": {53            "from": "scene_name"54          },55          "dataset_source": {56            "from": "dataset"57          },58          "question_type": {59            "from": "question_type"60          }61        },62        "source": {63          "format": "huggingface",64          "url": {65            "test": "https://huggingface.co/datasets/nyu-visionx/VSI-Bench (QA metadata, 'full' config); videos mirrored from https://huggingface.co/datasets/lccshunli/VSIBench"66          }67        }68      }69    }70  }71}