mm-eval/MMVet-v2

1{2  "name": "MMVet-v2",3  "release_date": "2024-08-01",4  "subsets": {5    "main": {6      "language": [7        "en"8      ],9      "modalities": [10        "multi_image_interleave"11      ],12      "task_type": "open_ended_qa",13      "score_pipeline": [14        "llm-judge"15      ],16      "score_params": {17        "rubric": "partial_credit",18        "scale": [19          0.0,20          1.021        ],22        "official_judge_model": "gpt-4-0613",23        "reference_syntax": "and_or_composition"24      },25      "score_protocol": {26        "reference": "lmms-eval@lmms_eval/tasks/mmvetv2/utils.py:193-245 (MM_VET_PROMPT, 0.0..1.0 partial credit with <AND>/<OR>) + lmms_eval/tasks/mmvetv2/mmvetv2.yaml:18 (gpt_eval_score); official whyu/mm-vet-v2 uses the MM-Vet GPT-4 evaluator. Verified in local clone: MM_VET_PROMPT at utils.py:193, yaml metric gpt_eval_score at line 18.",27        "note": "Official headline = mean judge score x100 overall + per-capability. DUPLICATE UPLOAD of mm-eval/MM-Vet-v2 (identical 517 rows, newer conversion vintage)."28      },29      "prompt_template": "{{ question }}",30      "mapping_from_source": {31        "media": {32          "from": "images",33          "type": "list",34          "min_items": 1,35          "max_items": 1836        },37        "id": {38          "from": "id"39        },40        "question": {41          "from": "question"42        },43        "answer": {44          "from": "answer",45          "optional": true46        },47        "extra": {48          "capability": {49            "from": "capability"50          },51          "added_in": {52            "from": "added_in"53          },54          "n_images": {55            "from": "n_images"56          }57        },58        "source": {59          "format": "huggingface",60          "url": {61            "test": "https://huggingface.co/datasets/whyu/mm-vet-v2"62          }63        }64      },65      "prompt_template_source": {66        "origin": "official",67        "reference": "https://github.com/open-compass/VLMEvalKit/blob/main/vlmeval/dataset/image_vqa.py (MMVet-v2 — judge-scored; bare question)",68        "notes": "Tier 3: MMVet-v2 judge-scored eval: bare question per the official MMVet-v2 paper."69      }70    }71  }72}