AnTo2209/MarineEVT
MarineEVT Dataset MarineEVT: Advancing Event-Centric Marine Video Understanding via Visual Tool Reasoning 📖 Description MarineEVT is a comprehensive event-centric dataset and benchmark for marine video understanding. It comprises 20,000 richly annotated underwater video question-answer pairs spanning 20 fine-grained dimensions, designed to support semantic, contextualized, spatial-temporal, and causal reasoning in marine environments. The dataset addresses the… See the full description on the dataset page: https://huggingface.co/datasets/AnTo2209/MarineEVT.
01.5k
1{2 "data": [3 {4 "question": "Is starfish positioned closer to our perspective than sponge based on their relative depth in the video? Answer yes or no only.",5 "answer": "yes",6 "explanation": {7 "0": {8 "sponge": [9 "1.945803165435791",10 "178.78561401367188",11 "1079.772705078125",12 "1472.0814208984375"13 ],14 "starfish": [15 "58.74114990234375",16 "420.1004333496094",17 "943.771240234375",18 "1518.047119140625"19 ]20 },21 "1": {22 "sponge": [23 "0.844799280166626",24 "179.4633331298828",25 "1078.3115234375",26 "1500.2540283203125"27 ],28 "starfish": [29 "104.34878540039062",30 "278.0762023925781",31 "905.042236328125",32 "1411.250732421875"33 ]34 },35 "2": {36 "sponge": [37 "2.3794519901275635",38 "179.31878662109375",39 "1080.24951171875",40 "1499.2550048828125"41 ],42 "starfish": [43 "193.84524536132812",44 "402.7083740234375",45 "840.0046997070312",46 "1472.3682861328125"47 ]48 },49 "3": {50 "sponge": [51 "2.2314906120300293",52 "178.67111206054688",53 "1079.9417724609375",54 "1499.3602294921875"55 ],56 "starfish": [57 "130.44281005859375",58 "426.2225036621094",59 "831.4177856445312",60 "1520.45068359375"61 ]62 },63 "4": {64 "sponge": [65 "0.5563759803771973",66 "179.29029846191406",67 "1078.2470703125",68 "1487.212646484375"69 ],70 "starfish": [71 "70.63992309570312",72 "564.3818359375",73 "815.7774658203125",74 "1497.494873046875"75 ]76 },77 "5": {78 "sponge": [79 "1.6563177108764648",80 "178.59747314453125",81 "1079.554443359375",82 "1370.994140625"83 ],84 "starfish": [85 "-1.6732478141784668",86 "478.31707763671875",87 "724.3530883789062",88 "1503.749267578125"89 ]90 },91 "6": {92 "sponge": [93 "1.2050950527191162",94 "176.9039764404297",95 "1078.9793701171875",96 "1312.1507568359375"97 ],98 "starfish": [99 "-2.887387275695801",100 "395.3875427246094",101 "665.34912109375",102 "1455.2716064453125"103 ]104 },105 "7": {106 "sponge": [107 "517.5340576171875",108 "265.8253173828125",109 "967.6988525390625",110 "744.116943359375"111 ],112 "starfish": [113 "12.076570510864258",114 "181.32545471191406",115 "578.4891357421875",116 "752.6017456054688"117 ]118 },119 "8": {120 "sponge": [121 "391.38360595703125",122 "185.9326171875",123 "933.5377197265625",124 "701.9183349609375"125 ],126 "starfish": [127 "-1.9107681512832642",128 "180.28509521484375",129 "446.98284912109375",130 "701.1033935546875"131 ]132 }133 },134 "dimension": "SpatialReasoning",135 "subdimension": "ProximityAssessment",136 "question_task": "VideoQuestionAnswering",137 "question_format": "ExactMatchQuestion",138 "video_id": "QP8h0Qh1NF3e",139 "video_url": "https://www.instagram.com/onebreathdiver/reel/Cf3lc_Xqyt0",140 "video_scene": "000026_000045",141 "id": 0,142 "original_video_id": "Cf3lc_Xqyt0",143 "turns": [144 {145 "turn_id": 1,146 "id": 0,147 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of starfish positioned closer to our perspective than sponge based on their relative depth in the video\", \"ground_type\": \"anything\"}}}]",148 "user_query": "Is starfish positioned closer to our perspective than sponge based on their relative depth in the video? Answer yes or no only.",149 "visual_input": [150 "frames/frame_000000.jpg",151 "frames/frame_000001.jpg",152 "frames/frame_000002.jpg",153 "frames/frame_000003.jpg",154 "frames/frame_000004.jpg",155 "frames/frame_000005.jpg",156 "frames/frame_000006.jpg",157 "frames/frame_000007.jpg",158 "frames/frame_000008.jpg"159 ],160 "tool_output": "{\"boxes\": [[[\"1.945803165435791\", \"178.78561401367188\", \"1079.772705078125\", \"1472.0814208984375\"], [\"58.74114990234375\", \"420.1004333496094\", \"943.771240234375\", \"1518.047119140625\"]], [[\"0.844799280166626\", \"179.4633331298828\", \"1078.3115234375\", \"1500.2540283203125\"], [\"104.34878540039062\", \"278.0762023925781\", \"905.042236328125\", \"1411.250732421875\"]], [[\"2.3794519901275635\", \"179.31878662109375\", \"1080.24951171875\", \"1499.2550048828125\"], [\"193.84524536132812\", \"402.7083740234375\", \"840.0046997070312\", \"1472.3682861328125\"]], [[\"2.2314906120300293\", \"178.67111206054688\", \"1079.9417724609375\", \"1499.3602294921875\"], [\"130.44281005859375\", \"426.2225036621094\", \"831.4177856445312\", \"1520.45068359375\"]], [[\"0.5563759803771973\", \"179.29029846191406\", \"1078.2470703125\", \"1487.212646484375\"], [\"70.63992309570312\", \"564.3818359375\", \"815.7774658203125\", \"1497.494873046875\"]], [[\"1.6563177108764648\", \"178.59747314453125\", \"1079.554443359375\", \"1370.994140625\"], [\"-1.6732478141784668\", \"478.31707763671875\", \"724.3530883789062\", \"1503.749267578125\"]], [[\"1.2050950527191162\", \"176.9039764404297\", \"1078.9793701171875\", \"1312.1507568359375\"], [\"-2.887387275695801\", \"395.3875427246094\", \"665.34912109375\", \"1455.2716064453125\"]], [[\"517.5340576171875\", \"265.8253173828125\", \"967.6988525390625\", \"744.116943359375\"], [\"12.076570510864258\", \"181.32545471191406\", \"578.4891357421875\", \"752.6017456054688\"]], [[\"391.38360595703125\", \"185.9326171875\", \"933.5377197265625\", \"701.9183349609375\"], [\"-1.9107681512832642\", \"180.28509521484375\", \"446.98284912109375\", \"701.1033935546875\"]]]}"161 },162 {163 "id": 0,164 "turn_id": 2,165 "user_query": "Is starfish positioned closer to our perspective than sponge based on their relative depth in the video? Answer yes or no only.",166 "assistant_response": "<answer>yes</answer>",167 "visual_input": [168 "frames/frame_000000.jpg",169 "frames/frame_000001.jpg",170 "frames/frame_000002.jpg",171 "frames/frame_000003.jpg",172 "frames/frame_000004.jpg",173 "frames/frame_000005.jpg",174 "frames/frame_000006.jpg",175 "frames/frame_000007.jpg",176 "frames/frame_000008.jpg"177 ]178 }179 ]180 },181 {182 "question": "Is conch shell positioned farther to our perspective than octopus based on their relative depth in the video? Answer yes or no only.",183 "answer": "yes",184 "explanation": {185 "0": {186 "octopus": [187 "342.483154296875",188 "102.14933776855469",189 "748.2162475585938",190 "565.4522094726562"191 ],192 "conch shell": [193 "326.4022216796875",194 "176.8466033935547",195 "878.10986328125",196 "484.78460693359375"197 ]198 },199 "1": {200 "octopus": [201 "365.4928894042969",202 "156.55010986328125",203 "725.7994384765625",204 "540.56005859375"205 ],206 "conch shell": [207 "355.023193359375",208 "180.21487426757812",209 "883.0126953125",210 "468.6161193847656"211 ]212 },213 "2": {214 "octopus": [215 "382.0979309082031",216 "206.51898193359375",217 "711.852783203125",218 "547.008056640625"219 ],220 "conch shell": [221 "361.069091796875",222 "196.10308837890625",223 "881.0291748046875",224 "480.11956787109375"225 ]226 },227 "3": {228 "octopus": [229 "368.08795166015625",230 "192.24241638183594",231 "686.4112548828125",232 "552.515380859375"233 ],234 "conch shell": [235 "348.3673095703125",236 "192.15036010742188",237 "873.0614013671875",238 "479.5621337890625"239 ]240 }241 },242 "dimension": "SpatialReasoning",243 "subdimension": "ProximityAssessment",244 "question_task": "VideoQuestionAnswering",245 "question_format": "ExactMatchQuestion",246 "video_id": "RHgaUotaRkRw",247 "video_url": "https://www.instagram.com/onebreathdiver/reel/CKZrODUBs8V",248 "video_scene": "000470_000473",249 "id": 1,250 "original_video_id": "CKZrODUBs8V",251 "turns": [252 {253 "turn_id": 1,254 "id": 1,255 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of conch shell positioned farther to our perspective than octopus\", \"ground_type\": \"anything\"}}}]",256 "user_query": "Is conch shell positioned farther to our perspective than octopus based on their relative depth in the video? Answer yes or no only.",257 "visual_input": [258 "frames/frame_000000.jpg",259 "frames/frame_000001.jpg",260 "frames/frame_000002.jpg",261 "frames/frame_000003.jpg"262 ],263 "tool_output": "{\"boxes\": [[[\"342.483154296875\", \"102.14933776855469\", \"748.2162475585938\", \"565.4522094726562\"], [\"326.4022216796875\", \"176.8466033935547\", \"878.10986328125\", \"484.78460693359375\"]], [[\"365.4928894042969\", \"156.55010986328125\", \"725.7994384765625\", \"540.56005859375\"], [\"355.023193359375\", \"180.21487426757812\", \"883.0126953125\", \"468.6161193847656\"]], [[\"382.0979309082031\", \"206.51898193359375\", \"711.852783203125\", \"547.008056640625\"], [\"361.069091796875\", \"196.10308837890625\", \"881.0291748046875\", \"480.11956787109375\"]], [[\"368.08795166015625\", \"192.24241638183594\", \"686.4112548828125\", \"552.515380859375\"], [\"348.3673095703125\", \"192.15036010742188\", \"873.0614013671875\", \"479.5621337890625\"]]]}"264 },265 {266 "id": 1,267 "turn_id": 2,268 "user_query": "Is conch shell positioned farther to our perspective than octopus based on their relative depth in the video? Answer yes or no only.",269 "assistant_response": "<answer>yes</answer>",270 "visual_input": [271 "frames/frame_000000.jpg",272 "frames/frame_000001.jpg",273 "frames/frame_000002.jpg",274 "frames/frame_000003.jpg"275 ]276 }277 ]278 },279 {280 "question": "Is sponge positioned closer to our perspective than small reef fish based on their relative depth in the video? Answer yes or no only.",281 "answer": "no",282 "explanation": {283 "0": {284 "small reef fish": [285 "109.90447998046875",286 "174.689697265625",287 "314.1343688964844",288 "340.5445861816406"289 ],290 "sponge": [291 "-0.4986119270324707",292 "36.28173828125",293 "351.37457275390625",294 "276.2792053222656"295 ]296 },297 "1": {298 "small reef fish": [299 "116.22718811035156",300 "166.25216674804688",301 "325.05267333984375",302 "348.01495361328125"303 ],304 "sponge": [305 "-0.31319618225097656",306 "35.056541442871094",307 "349.18988037109375",308 "275.84619140625"309 ]310 },311 "2": {312 "small reef fish": [313 "118.47590637207031",314 "147.57325744628906",315 "337.7566833496094",316 "340.8617858886719"317 ],318 "sponge": [319 "-0.6934690475463867",320 "11.41143798828125",321 "355.0309753417969",322 "273.7175598144531"323 ]324 }325 },326 "dimension": "SpatialReasoning",327 "subdimension": "ProximityAssessment",328 "question_task": "VideoQuestionAnswering",329 "question_format": "ExactMatchQuestion",330 "video_id": "jUxscI8UVQ94",331 "video_url": "https://www.instagram.com/dive_down_under/reel/DBpAre8Iln3",332 "video_scene": "000029_000032",333 "id": 2,334 "original_video_id": "DBpAre8Iln3",335 "turns": [336 {337 "turn_id": 1,338 "id": 2,339 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of sponge positioned closer to our perspective than small reef fish based on their relative depth in the video\", \"ground_type\": \"anything\"}}}]",340 "user_query": "Is sponge positioned closer to our perspective than small reef fish based on their relative depth in the video? Answer yes or no only.",341 "visual_input": [342 "frames/frame_000000.jpg",343 "frames/frame_000001.jpg",344 "frames/frame_000002.jpg"345 ],346 "tool_output": "{\"boxes\": [[[\"109.90447998046875\", \"174.689697265625\", \"314.1343688964844\", \"340.5445861816406\"], [\"-0.4986119270324707\", \"36.28173828125\", \"351.37457275390625\", \"276.2792053222656\"]], [[\"116.22718811035156\", \"166.25216674804688\", \"325.05267333984375\", \"348.01495361328125\"], [\"-0.31319618225097656\", \"35.056541442871094\", \"349.18988037109375\", \"275.84619140625\"]], [[\"118.47590637207031\", \"147.57325744628906\", \"337.7566833496094\", \"340.8617858886719\"], [\"-0.6934690475463867\", \"11.41143798828125\", \"355.0309753417969\", \"273.7175598144531\"]]]}"347 },348 {349 "id": 2,350 "turn_id": 2,351 "user_query": "Is sponge positioned closer to our perspective than small reef fish based on their relative depth in the video? Answer yes or no only.",352 "assistant_response": "<answer>no</answer>",353 "visual_input": [354 "frames/frame_000000.jpg",355 "frames/frame_000001.jpg",356 "frames/frame_000002.jpg"357 ]358 }359 ]360 },361 {362 "question": "Is clownfish positioned closer to our perspective than coral based on their relative depth in the video? Answer yes or no only.",363 "answer": "no",364 "explanation": {365 "0": {366 "coral": [367 "0.2559471130371094",368 "410.833740234375",369 "359.36541748046875",370 "639.9686279296875"371 ],372 "clownfish": [373 "-1.9881927967071533",374 "225.96786499023438",375 "118.27904510498047",376 "308.0655212402344"377 ]378 },379 "1": {380 "coral": [381 "0.04980325698852539",382 "358.0230712890625",383 "359.3628234863281",384 "639.5615234375"385 ],386 "clownfish": [387 "10.635141372680664",388 "201.02890014648438",389 "142.58387756347656",390 "280.478515625"391 ]392 },393 "2": {394 "coral": [395 "0.8258414268493652",396 "349.6470947265625",397 "360.04119873046875",398 "639.76220703125"399 ],400 "clownfish": [401 "-1.0181772708892822",402 "188.29464721679688",403 "141.78399658203125",404 "256.7867431640625"405 ]406 },407 "3": {408 "coral": [409 "0.8713424205780029",410 "373.57757568359375",411 "359.8275146484375",412 "639.472412109375"413 ],414 "clownfish": [415 "0.13228118419647217",416 "223.114501953125",417 "115.15544891357422",418 "310.974853515625"419 ]420 }421 },422 "dimension": "SpatialReasoning",423 "subdimension": "ProximityAssessment",424 "question_task": "VideoQuestionAnswering",425 "question_format": "ExactMatchQuestion",426 "video_id": "jvCIVj3vfqT5",427 "video_url": "https://www.instagram.com/dive_down_under/reel/DC7aIHKzV8r",428 "video_scene": "000000_000006",429 "id": 3,430 "original_video_id": "DC7aIHKzV8r",431 "turns": [432 {433 "turn_id": 1,434 "id": 3,435 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of clownfish positioned closer to our perspective than coral based on their relative depth in the video\", \"ground_type\": \"anything\"}}}]",436 "user_query": "Is clownfish positioned closer to our perspective than coral based on their relative depth in the video? Answer yes or no only.",437 "visual_input": [438 "frames/frame_000000.jpg",439 "frames/frame_000001.jpg",440 "frames/frame_000002.jpg",441 "frames/frame_000003.jpg"442 ],443 "tool_output": "{\"boxes\": [[[\"0.2559471130371094\", \"410.833740234375\", \"359.36541748046875\", \"639.9686279296875\"], [\"-1.9881927967071533\", \"225.96786499023438\", \"118.27904510498047\", \"308.0655212402344\"]], [[\"0.04980325698852539\", \"358.0230712890625\", \"359.3628234863281\", \"639.5615234375\"], [\"10.635141372680664\", \"201.02890014648438\", \"142.58387756347656\", \"280.478515625\"]], [[\"0.8258414268493652\", \"349.6470947265625\", \"360.04119873046875\", \"639.76220703125\"], [\"-1.0181772708892822\", \"188.29464721679688\", \"141.78399658203125\", \"256.7867431640625\"]], [[\"0.8713424205780029\", \"373.57757568359375\", \"359.8275146484375\", \"639.472412109375\"], [\"0.13228118419647217\", \"223.114501953125\", \"115.15544891357422\", \"310.974853515625\"]]]}"444 },445 {446 "id": 3,447 "turn_id": 2,448 "user_query": "Is clownfish positioned closer to our perspective than coral based on their relative depth in the video? Answer yes or no only.",449 "assistant_response": "<answer>no</answer>",450 "visual_input": [451 "frames/frame_000000.jpg",452 "frames/frame_000001.jpg",453 "frames/frame_000002.jpg",454 "frames/frame_000003.jpg"455 ]456 }457 ]458 },459 {460 "question": "Is giant clam positioned farther to our perspective than coral (soft coral) based on their relative depth in the video? Answer yes or no only.",461 "answer": "no",462 "explanation": {463 "0": {464 "coral (soft coral)": [465 "34.6834716796875",466 "514.1779174804688",467 "670.9620361328125",468 "1246.802734375"469 ],470 "giant clam": [471 "233.98886108398438",472 "522.2757568359375",473 "669.205078125",474 "1245.6824951171875"475 ]476 },477 "1": {478 "coral (soft coral)": [479 "37.25584030151367",480 "513.2987060546875",481 "672.8770141601562",482 "1247.3641357421875"483 ],484 "giant clam": [485 "230.8139190673828",486 "519.226806640625",487 "671.5902099609375",488 "1245.6090087890625"489 ]490 },491 "2": {492 "coral (soft coral)": [493 "36.308292388916016",494 "511.83343505859375",495 "671.3402099609375",496 "1246.259521484375"497 ],498 "giant clam": [499 "228.73123168945312",500 "520.6299438476562",501 "670.5955200195312",502 "1244.386474609375"503 ]504 },505 "3": {506 "coral (soft coral)": [507 "35.64490509033203",508 "508.37091064453125",509 "672.6223754882812",510 "1245.303466796875"511 ],512 "giant clam": [513 "36.16671371459961",514 "509.14434814453125",515 "671.8192749023438",516 "1246.9100341796875"517 ]518 },519 "4": {520 "coral (soft coral)": [521 "39.743858337402344",522 "510.45660400390625",523 "673.4693603515625",524 "1245.8890380859375"525 ],526 "giant clam": [527 "232.37741088867188",528 "521.3692626953125",529 "672.1470947265625",530 "1245.0029296875"531 ]532 }533 },534 "dimension": "SpatialReasoning",535 "subdimension": "ProximityAssessment",536 "question_task": "VideoQuestionAnswering",537 "question_format": "ExactMatchQuestion",538 "video_id": "AaRPK5ZMFNwY",539 "video_url": "https://www.instagram.com/dive_down_under/reel/DRc_wSqE300",540 "video_scene": "000000_000010",541 "id": 4,542 "original_video_id": "DRc_wSqE300",543 "turns": [544 {545 "turn_id": 1,546 "id": 4,547 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of giant clam positioned farther to our perspective than coral (soft coral)\", \"ground_type\": \"anything\"}}}]",548 "user_query": "Is giant clam positioned farther to our perspective than coral (soft coral) based on their relative depth in the video? Answer yes or no only.",549 "visual_input": [550 "frames/frame_000000.jpg",551 "frames/frame_000001.jpg",552 "frames/frame_000002.jpg",553 "frames/frame_000003.jpg",554 "frames/frame_000004.jpg"555 ],556 "tool_output": "{\"boxes\": [[[\"34.6834716796875\", \"514.1779174804688\", \"670.9620361328125\", \"1246.802734375\"], [\"233.98886108398438\", \"522.2757568359375\", \"669.205078125\", \"1245.6824951171875\"]], [[\"37.25584030151367\", \"513.2987060546875\", \"672.8770141601562\", \"1247.3641357421875\"], [\"230.8139190673828\", \"519.226806640625\", \"671.5902099609375\", \"1245.6090087890625\"]], [[\"36.308292388916016\", \"511.83343505859375\", \"671.3402099609375\", \"1246.259521484375\"], [\"228.73123168945312\", \"520.6299438476562\", \"670.5955200195312\", \"1244.386474609375\"]], [[\"35.64490509033203\", \"508.37091064453125\", \"672.6223754882812\", \"1245.303466796875\"], [\"36.16671371459961\", \"509.14434814453125\", \"671.8192749023438\", \"1246.9100341796875\"]], [[\"39.743858337402344\", \"510.45660400390625\", \"673.4693603515625\", \"1245.8890380859375\"], [\"232.37741088867188\", \"521.3692626953125\", \"672.1470947265625\", \"1245.0029296875\"]]]}"557 },558 {559 "id": 4,560 "turn_id": 2,561 "user_query": "Is giant clam positioned farther to our perspective than coral (soft coral) based on their relative depth in the video? Answer yes or no only.",562 "assistant_response": "<answer>no</answer>",563 "visual_input": [564 "frames/frame_000000.jpg",565 "frames/frame_000001.jpg",566 "frames/frame_000002.jpg",567 "frames/frame_000003.jpg",568 "frames/frame_000004.jpg"569 ]570 }571 ]572 },573 {574 "question": "Is wooden pilings / submerged structure positioned farther to our perspective than seagrass based on their relative depth in the video? Answer yes or no only.",575 "answer": "yes",576 "explanation": {577 "0": {578 "seagrass": [579 "-0.2440810203552246",580 "95.28712463378906",581 "718.9073486328125",582 "539.1123046875"583 ],584 "wooden pilings / submerged structure": [585 "400.92742919921875",586 "-0.522724986076355",587 "542.7216186523438",588 "438.17095947265625"589 ]590 },591 "1": {592 "seagrass": [593 "0.8956646919250488",594 "151.51925659179688",595 "719.2935180664062",596 "537.994384765625"597 ],598 "wooden pilings / submerged structure": [599 "433.0276184082031",600 "0.7572680711746216",601 "588.607666015625",602 "469.094482421875"603 ]604 },605 "2": {606 "seagrass": [607 "-0.3937697410583496",608 "169.1327362060547",609 "702.4609985351562",610 "540.1416015625"611 ],612 "wooden pilings / submerged structure": [613 "453.1905822753906",614 "-0.08473098278045654",615 "710.5631713867188",616 "536.7056274414062"617 ]618 },619 "3": {620 "seagrass": [621 "1.54433012008667",622 "205.04759216308594",623 "719.533447265625",624 "539.7523803710938"625 ],626 "wooden pilings / submerged structure": [627 "500.2431335449219",628 "-0.22709190845489502",629 "719.1976318359375",630 "535.5410766601562"631 ]632 },633 "4": {634 "seagrass": [635 "0.5473637580871582",636 "215.85897827148438",637 "718.8775024414062",638 "539.0545654296875"639 ],640 "wooden pilings / submerged structure": [641 "566.83251953125",642 "0.03770649433135986",643 "722.2974243164062",644 "375.46649169921875"645 ]646 },647 "5": {648 "seagrass": [649 "0.6010723114013672",650 "321.7590026855469",651 "718.6636962890625",652 "539.508056640625"653 ],654 "wooden pilings / submerged structure": [655 "191.66468811035156",656 "174.4219970703125",657 "231.71006774902344",658 "324.7815246582031"659 ]660 }661 },662 "dimension": "SpatialReasoning",663 "subdimension": "ProximityAssessment",664 "question_task": "VideoQuestionAnswering",665 "question_format": "ExactMatchQuestion",666 "video_id": "g7ac7UnJpWVt",667 "video_url": "https://www.instagram.com/onebreathdiver/reel/CQGE3ckhO_f",668 "video_scene": "000001_000019",669 "id": 5,670 "original_video_id": "CQGE3ckhO_f",671 "turns": [672 {673 "turn_id": 1,674 "id": 5,675 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of wooden pilings submerged structure positioned farther to our perspective than seagrass\", \"ground_type\": \"anything\"}}}]",676 "user_query": "Is wooden pilings / submerged structure positioned farther to our perspective than seagrass based on their relative depth in the video? Answer yes or no only.",677 "visual_input": [678 "frames/frame_000000.jpg",679 "frames/frame_000001.jpg",680 "frames/frame_000002.jpg",681 "frames/frame_000003.jpg",682 "frames/frame_000004.jpg",683 "frames/frame_000005.jpg"684 ],685 "tool_output": "{\"boxes\": [[[\"-0.2440810203552246\", \"95.28712463378906\", \"718.9073486328125\", \"539.1123046875\"], [\"400.92742919921875\", \"-0.522724986076355\", \"542.7216186523438\", \"438.17095947265625\"]], [[\"0.8956646919250488\", \"151.51925659179688\", \"719.2935180664062\", \"537.994384765625\"], [\"433.0276184082031\", \"0.7572680711746216\", \"588.607666015625\", \"469.094482421875\"]], [[\"-0.3937697410583496\", \"169.1327362060547\", \"702.4609985351562\", \"540.1416015625\"], [\"453.1905822753906\", \"-0.08473098278045654\", \"710.5631713867188\", \"536.7056274414062\"]], [[\"1.54433012008667\", \"205.04759216308594\", \"719.533447265625\", \"539.7523803710938\"], [\"500.2431335449219\", \"-0.22709190845489502\", \"719.1976318359375\", \"535.5410766601562\"]], [[\"0.5473637580871582\", \"215.85897827148438\", \"718.8775024414062\", \"539.0545654296875\"], [\"566.83251953125\", \"0.03770649433135986\", \"722.2974243164062\", \"375.46649169921875\"]], [[\"0.6010723114013672\", \"321.7590026855469\", \"718.6636962890625\", \"539.508056640625\"], [\"191.66468811035156\", \"174.4219970703125\", \"231.71006774902344\", \"324.7815246582031\"]]]}"686 },687 {688 "id": 5,689 "turn_id": 2,690 "user_query": "Is wooden pilings / submerged structure positioned farther to our perspective than seagrass based on their relative depth in the video? Answer yes or no only.",691 "assistant_response": "<answer>yes</answer>",692 "visual_input": [693 "frames/frame_000000.jpg",694 "frames/frame_000001.jpg",695 "frames/frame_000002.jpg",696 "frames/frame_000003.jpg",697 "frames/frame_000004.jpg",698 "frames/frame_000005.jpg"699 ]700 }701 ]702 },703 {704 "question": "Is scallop positioned farther to our perspective than octopus based on their relative depth in the video? Answer yes or no only.",705 "answer": "yes",706 "explanation": {707 "0": {708 "octopus": [709 "1.048121452331543",710 "290.28350830078125",711 "717.287353515625",712 "894.1494750976562"713 ],714 "scallop": [715 "74.95829772949219",716 "483.50146484375",717 "569.6857299804688",718 "763.05859375"719 ]720 },721 "1": {722 "octopus": [723 "1.2830829620361328",724 "310.12078857421875",725 "718.0348510742188",726 "927.4201049804688"727 ],728 "scallop": [729 "44.24795150756836",730 "491.59906005859375",731 "541.3596801757812",732 "783.53369140625"733 ]734 },735 "2": {736 "octopus": [737 "1.4458179473876953",738 "320.3950500488281",739 "718.274169921875",740 "869.3792724609375"741 ],742 "scallop": [743 "110.51121520996094",744 "480.1081848144531",745 "574.5018920898438",746 "761.947509765625"747 ]748 }749 },750 "dimension": "SpatialReasoning",751 "subdimension": "ProximityAssessment",752 "question_task": "VideoQuestionAnswering",753 "question_format": "ExactMatchQuestion",754 "video_id": "6tU0iin9rgEg",755 "video_url": "https://www.instagram.com/onebreathdiver/reel/CiUH4zco8Ih",756 "video_scene": "000014_000016",757 "id": 6,758 "original_video_id": "CiUH4zco8Ih",759 "turns": [760 {761 "turn_id": 1,762 "id": 6,763 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of scallop positioned farther to our perspective than octopus\", \"ground_type\": \"anything\"}}}]",764 "user_query": "Is scallop positioned farther to our perspective than octopus based on their relative depth in the video? Answer yes or no only.",765 "visual_input": [766 "frames/frame_000000.jpg",767 "frames/frame_000001.jpg",768 "frames/frame_000002.jpg"769 ],770 "tool_output": "{\"boxes\": [[[\"1.048121452331543\", \"290.28350830078125\", \"717.287353515625\", \"894.1494750976562\"], [\"74.95829772949219\", \"483.50146484375\", \"569.6857299804688\", \"763.05859375\"]], [[\"1.2830829620361328\", \"310.12078857421875\", \"718.0348510742188\", \"927.4201049804688\"], [\"44.24795150756836\", \"491.59906005859375\", \"541.3596801757812\", \"783.53369140625\"]], [[\"1.4458179473876953\", \"320.3950500488281\", \"718.274169921875\", \"869.3792724609375\"], [\"110.51121520996094\", \"480.1081848144531\", \"574.5018920898438\", \"761.947509765625\"]]]}"771 },772 {773 "id": 6,774 "turn_id": 2,775 "user_query": "Is scallop positioned farther to our perspective than octopus based on their relative depth in the video? Answer yes or no only.",776 "assistant_response": "<answer>yes</answer>",777 "visual_input": [778 "frames/frame_000000.jpg",779 "frames/frame_000001.jpg",780 "frames/frame_000002.jpg"781 ]782 }783 ]784 },785 {786 "question": "Is coral reef structure positioned closer to our perspective than red fish based on their relative depth in the video? Answer yes or no only.",787 "answer": "yes",788 "explanation": {789 "0": {790 "red fish": [791 "620.7723388671875",792 "828.9276733398438",793 "722.579833984375",794 "989.2864379882812"795 ],796 "coral reef structure": [797 "-0.902552604675293",798 "-2.908935546875",799 "718.3749389648438",800 "1273.7890625"801 ]802 },803 "1": {804 "red fish": [805 "581.1141357421875",806 "878.490478515625",807 "723.5532836914062",808 "1019.110595703125"809 ],810 "coral reef structure": [811 "0.6592655181884766",812 "-2.1851730346679688",813 "719.87451171875",814 "1273.8037109375"815 ]816 },817 "2": {818 "red fish": [819 "-1.7768508195877075",820 "198.6381378173828",821 "76.17743682861328",822 "330.16265869140625"823 ],824 "coral reef structure": [825 "-0.5796146392822266",826 "-0.45063018798828125",827 "718.8103637695312",828 "1275.1641845703125"829 ]830 },831 "3": {832 "red fish": [833 "-0.9631222486495972",834 "199.439208984375",835 "124.73147583007812",836 "378.1077575683594"837 ],838 "coral reef structure": [839 "-0.6226587295532227",840 "0.556488037109375",841 "718.7380981445312",842 "1276.5166015625"843 ]844 }845 },846 "dimension": "SpatialReasoning",847 "subdimension": "ProximityAssessment",848 "question_task": "VideoQuestionAnswering",849 "question_format": "ExactMatchQuestion",850 "video_id": "yzaoxWg8ZXCC",851 "video_url": "https://www.instagram.com/romanhurghada/reel/DIcJ056tr4_",852 "video_scene": "000018_000030",853 "id": 7,854 "original_video_id": "DIcJ056tr4_",855 "turns": [856 {857 "turn_id": 1,858 "id": 7,859 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of coral reef structure positioned closer to our perspective than red fish based on their relative depth in the video\", \"ground_type\": \"anything\"}}}]",860 "user_query": "Is coral reef structure positioned closer to our perspective than red fish based on their relative depth in the video? Answer yes or no only.",861 "visual_input": [862 "frames/frame_000000.jpg",863 "frames/frame_000001.jpg",864 "frames/frame_000002.jpg",865 "frames/frame_000003.jpg"866 ],867 "tool_output": "{\"boxes\": [[[\"620.7723388671875\", \"828.9276733398438\", \"722.579833984375\", \"989.2864379882812\"], [\"-0.902552604675293\", \"-2.908935546875\", \"718.3749389648438\", \"1273.7890625\"]], [[\"581.1141357421875\", \"878.490478515625\", \"723.5532836914062\", \"1019.110595703125\"], [\"0.6592655181884766\", \"-2.1851730346679688\", \"719.87451171875\", \"1273.8037109375\"]], [[\"-1.7768508195877075\", \"198.6381378173828\", \"76.17743682861328\", \"330.16265869140625\"], [\"-0.5796146392822266\", \"-0.45063018798828125\", \"718.8103637695312\", \"1275.1641845703125\"]], [[\"-0.9631222486495972\", \"199.439208984375\", \"124.73147583007812\", \"378.1077575683594\"], [\"-0.6226587295532227\", \"0.556488037109375\", \"718.7380981445312\", \"1276.5166015625\"]]]}"868 },869 {870 "id": 7,871 "turn_id": 2,872 "user_query": "Is coral reef structure positioned closer to our perspective than red fish based on their relative depth in the video? Answer yes or no only.",873 "assistant_response": "<answer>yes</answer>",874 "visual_input": [875 "frames/frame_000000.jpg",876 "frames/frame_000001.jpg",877 "frames/frame_000002.jpg",878 "frames/frame_000003.jpg"879 ]880 }881 ]882 },883 {884 "question": "Is sponge positioned farther to our perspective than giant moray eel based on their relative depth in the video? Answer yes or no only.",885 "answer": "yes",886 "explanation": {887 "0": {888 "giant moray eel": [889 "389.866455078125",890 "164.37948608398438",891 "722.5984497070312",892 "657.195556640625"893 ],894 "sponge": [895 "199.00735473632812",896 "1.91619873046875",897 "724.7425537109375",898 "1275.8690185546875"899 ]900 },901 "1": {902 "giant moray eel": [903 "378.1351318359375",904 "139.82662963867188",905 "726.72509765625",906 "677.4795532226562"907 ],908 "sponge": [909 "184.01136779785156",910 "3.9501190185546875",911 "723.9005126953125",912 "1276.6929931640625"913 ]914 },915 "2": {916 "giant moray eel": [917 "598.4652099609375",918 "285.2545471191406",919 "727.1212158203125",920 "771.6643676757812"921 ],922 "sponge": [923 "312.3742980957031",924 "1.7016983032226562",925 "724.0807495117188",926 "1276.9957275390625"927 ]928 }929 },930 "dimension": "SpatialReasoning",931 "subdimension": "ProximityAssessment",932 "question_task": "VideoQuestionAnswering",933 "question_format": "ExactMatchQuestion",934 "video_id": "bvaOH4MZSf2U",935 "video_url": "https://www.instagram.com/romanhurghada/reel/DNGT1jkNX_l",936 "video_scene": "000015_000023",937 "id": 8,938 "original_video_id": "DNGT1jkNX_l",939 "turns": [940 {941 "turn_id": 1,942 "id": 8,943 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of sponge positioned farther to our perspective than giant moray eel based on their relative depth in the video\", \"ground_type\": \"anything\"}}}]",944 "user_query": "Is sponge positioned farther to our perspective than giant moray eel based on their relative depth in the video? Answer yes or no only.",945 "visual_input": [946 "frames/frame_000000.jpg",947 "frames/frame_000001.jpg",948 "frames/frame_000002.jpg"949 ],950 "tool_output": "{\"boxes\": [[[\"389.866455078125\", \"164.37948608398438\", \"722.5984497070312\", \"657.195556640625\"], [\"199.00735473632812\", \"1.91619873046875\", \"724.7425537109375\", \"1275.8690185546875\"]], [[\"378.1351318359375\", \"139.82662963867188\", \"726.72509765625\", \"677.4795532226562\"], [\"184.01136779785156\", \"3.9501190185546875\", \"723.9005126953125\", \"1276.6929931640625\"]], [[\"598.4652099609375\", \"285.2545471191406\", \"727.1212158203125\", \"771.6643676757812\"], [\"312.3742980957031\", \"1.7016983032226562\", \"724.0807495117188\", \"1276.9957275390625\"]]]}"951 },952 {953 "id": 8,954 "turn_id": 2,955 "user_query": "Is sponge positioned farther to our perspective than giant moray eel based on their relative depth in the video? Answer yes or no only.",956 "assistant_response": "<answer>yes</answer>",957 "visual_input": [958 "frames/frame_000000.jpg",959 "frames/frame_000001.jpg",960 "frames/frame_000002.jpg"961 ]962 }963 ]964 },965 {966 "question": "Is lobster positioned farther to our perspective than fishing lure based on their relative depth in the video? Answer yes or no only.",967 "answer": "yes",968 "explanation": {969 "0": {970 "fishing lure": [971 "385.7417297363281",972 "401.6174011230469",973 "466.4540100097656",974 "676.2114868164062"975 ],976 "lobster": [977 "89.5341567993164",978 "218.4180145263672",979 "695.436767578125",980 "630.067138671875"981 ]982 },983 "1": {984 "fishing lure": [985 "377.6894836425781",986 "296.1540832519531",987 "455.6514892578125",988 "476.0233459472656"989 ],990 "lobster": [991 "113.32996368408203",992 "148.71087646484375",993 "644.7973022460938",994 "438.8987731933594"995 ]996 },997 "2": {998 "fishing lure": [999 "387.2373962402344",1000 "195.92684936523438",1001 "468.013671875",1002 "358.31524658203125"1003 ],1004 "lobster": [1005 "143.7864227294922",1006 "61.1094856262207",1007 "621.740234375",1008 "356.5133972167969"1009 ]1010 },1011 "3": {1012 "fishing lure": [1013 "357.65740966796875",1014 "148.82461547851562",1015 "424.10394287109375",1016 "294.9160461425781"1017 ],1018 "lobster": [1019 "128.2631072998047",1020 "17.911985397338867",1021 "571.2437133789062",1022 "288.46905517578125"1023 ]1024 },1025 "4": {1026 "fishing lure": [1027 "335.9471435546875",1028 "146.9468994140625",1029 "398.7806091308594",1030 "270.948974609375"1031 ],1032 "lobster": [1033 "137.054931640625",1034 "34.17677307128906",1035 "511.2414855957031",1036 "271.66326904296875"1037 ]1038 },1039 "5": {1040 "fishing lure": [1041 "346.9057922363281",1042 "291.6488952636719",1043 "403.3394775390625",1044 "437.89263916015625"1045 ],1046 "lobster": [1047 "132.0590057373047",1048 "186.5251922607422",1049 "529.736083984375",1050 "429.4213562011719"1051 ]1052 },1053 "6": {1054 "fishing lure": [1055 "411.6681823730469",1056 "462.91717529296875",1057 "540.5057983398438",1058 "677.0628662109375"1059 ],1060 "lobster": [1061 "180.8447265625",1062 "336.8385925292969",1063 "640.8728637695312",1064 "634.39501953125"1065 ]1066 },1067 "7": {1068 "fishing lure": [1069 "354.8698425292969",1070 "448.4844970703125",1071 "473.7621765136719",1072 "655.37646484375"1073 ],1074 "lobster": [1075 "122.34126281738281",1076 "324.60223388671875",1077 "591.1188354492188",1078 "626.445068359375"1079 ]1080 }1081 },1082 "dimension": "SpatialReasoning",1083 "subdimension": "ProximityAssessment",1084 "question_task": "VideoQuestionAnswering",1085 "question_format": "ExactMatchQuestion",1086 "video_id": "CdzxlQ0xVNrF",1087 "video_url": "https://www.instagram.com/onebreathdiver/reel/CdPEPmzKhJJ",1088 "video_scene": "000000_000007",1089 "id": 9,1090 "original_video_id": "CdPEPmzKhJJ",1091 "turns": [1092 {1093 "turn_id": 1,1094 "id": 9,1095 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of lobster positioned farther to our perspective than fishing lure\", \"ground_type\": \"anything\"}}}]",1096 "user_query": "Is lobster positioned farther to our perspective than fishing lure based on their relative depth in the video? Answer yes or no only.",1097 "visual_input": [1098 "frames/frame_000000.jpg",1099 "frames/frame_000001.jpg",1100 "frames/frame_000002.jpg",1101 "frames/frame_000003.jpg",1102 "frames/frame_000004.jpg",1103 "frames/frame_000005.jpg",1104 "frames/frame_000006.jpg",1105 "frames/frame_000007.jpg"1106 ],1107 "tool_output": "{\"boxes\": [[[\"385.7417297363281\", \"401.6174011230469\", \"466.4540100097656\", \"676.2114868164062\"], [\"89.5341567993164\", \"218.4180145263672\", \"695.436767578125\", \"630.067138671875\"]], [[\"377.6894836425781\", \"296.1540832519531\", \"455.6514892578125\", \"476.0233459472656\"], [\"113.32996368408203\", \"148.71087646484375\", \"644.7973022460938\", \"438.8987731933594\"]], [[\"387.2373962402344\", \"195.92684936523438\", \"468.013671875\", \"358.31524658203125\"], [\"143.7864227294922\", \"61.1094856262207\", \"621.740234375\", \"356.5133972167969\"]], [[\"357.65740966796875\", \"148.82461547851562\", \"424.10394287109375\", \"294.9160461425781\"], [\"128.2631072998047\", \"17.911985397338867\", \"571.2437133789062\", \"288.46905517578125\"]], [[\"335.9471435546875\", \"146.9468994140625\", \"398.7806091308594\", \"270.948974609375\"], [\"137.054931640625\", \"34.17677307128906\", \"511.2414855957031\", \"271.66326904296875\"]], [[\"346.9057922363281\", \"291.6488952636719\", \"403.3394775390625\", \"437.89263916015625\"], [\"132.0590057373047\", \"186.5251922607422\", \"529.736083984375\", \"429.4213562011719\"]], [[\"411.6681823730469\", \"462.91717529296875\", \"540.5057983398438\", \"677.0628662109375\"], [\"180.8447265625\", \"336.8385925292969\", \"640.8728637695312\", \"634.39501953125\"]], [[\"354.8698425292969\", \"448.4844970703125\", \"473.7621765136719\", \"655.37646484375\"], [\"122.34126281738281\", \"324.60223388671875\", \"591.1188354492188\", \"626.445068359375\"]]]}"1108 },1109 {1110 "id": 9,1111 "turn_id": 2,1112 "user_query": "Is lobster positioned farther to our perspective than fishing lure based on their relative depth in the video? Answer yes or no only.",1113 "assistant_response": "<answer>yes</answer>",1114 "visual_input": [1115 "frames/frame_000000.jpg",1116 "frames/frame_000001.jpg",1117 "frames/frame_000002.jpg",1118 "frames/frame_000003.jpg",1119 "frames/frame_000004.jpg",1120 "frames/frame_000005.jpg",1121 "frames/frame_000006.jpg",1122 "frames/frame_000007.jpg"1123 ]1124 }1125 ]1126 },1127 {1128 "question": "Is giant grouper positioned farther to our perspective than coral reef structure based on their relative depth in the video? Answer yes or no only.",1129 "answer": "no",1130 "explanation": {1131 "0": {1132 "coral reef structure": [1133 "-3.1984591484069824",1134 "180.9003143310547",1135 "398.2112121582031",1136 "721.46923828125"1137 ],1138 "giant grouper": [1139 "-2.25618839263916",1140 "103.32758331298828",1141 "337.2410888671875",1142 "445.6386413574219"1143 ]1144 },1145 "1": {1146 "coral reef structure": [1147 "-3.9896249771118164",1148 "153.55003356933594",1149 "405.1519775390625",1150 "719.4461059570312"1151 ],1152 "giant grouper": [1153 "-2.6875948905944824",1154 "62.741268157958984",1155 "333.2055969238281",1156 "433.18017578125"1157 ]1158 },1159 "2": {1160 "coral reef structure": [1161 "-3.1993603706359863",1162 "165.3442840576172",1163 "413.8899230957031",1164 "719.5509643554688"1165 ],1166 "giant grouper": [1167 "5.396293640136719",1168 "48.5056037902832",1169 "350.5671081542969",1170 "431.0884094238281"1171 ]1172 },1173 "3": {1174 "coral reef structure": [1175 "-7.3517632484436035",1176 "182.73654174804688",1177 "433.9725646972656",1178 "720.1405029296875"1179 ],1180 "giant grouper": [1181 "3.9679527282714844",1182 "44.788536071777344",1183 "371.2828369140625",1184 "445.4284973144531"1185 ]1186 },1187 "4": {1188 "coral reef structure": [1189 "-1.1760306358337402",1190 "209.46144104003906",1191 "446.4609375",1192 "720.0509033203125"1193 ],1194 "giant grouper": [1195 "23.280094146728516",1196 "49.443111419677734",1197 "391.6888732910156",1198 "472.3379211425781"1199 ]1200 },