AnTo2209/MarineEVT
MarineEVT Dataset MarineEVT: Advancing Event-Centric Marine Video Understanding via Visual Tool Reasoning 📖 Description MarineEVT is a comprehensive event-centric dataset and benchmark for marine video understanding. It comprises 20,000 richly annotated underwater video question-answer pairs spanning 20 fine-grained dimensions, designed to support semantic, contextualized, spatial-temporal, and causal reasoning in marine environments. The dataset addresses the… See the full description on the dataset page: https://huggingface.co/datasets/AnTo2209/MarineEVT.
01.5k
1{2 "data": [3 {4 "question": "What is the relative spatial relationship of stellate rabbitfish with respect to coral? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",5 "answer": "{\"x\": \"right\", \"y\": \"top\", \"z\": \"front\"}",6 "explanation": {7 "0": {8 "stellate rabbitfish": [9 "247.0327606201172",10 "341.46514892578125",11 "722.6377563476562",12 "755.986328125"13 ],14 "coral": [15 "3.3051037788391113",16 "358.26239013671875",17 "722.8549194335938",18 "1276.29052734375"19 ]20 },21 "1": {22 "stellate rabbitfish": [23 "213.98171997070312",24 "330.00787353515625",25 "724.9449462890625",26 "738.626953125"27 ],28 "coral": [29 "3.8461804389953613",30 "342.9699401855469",31 "723.3826904296875",32 "1276.8126220703125"33 ]34 },35 "2": {36 "stellate rabbitfish": [37 "209.9815673828125",38 "365.73577880859375",39 "718.5889892578125",40 "730.7915649414062"41 ],42 "coral": [43 "5.302748680114746",44 "321.976806640625",45 "724.8679809570312",46 "1277.0765380859375"47 ]48 }49 },50 "dimension": "SpatialReasoning",51 "subdimension": "SpatialRelationAssessment",52 "question_task": "VideoQuestionAnswering",53 "question_format": "ExactMatchQuestion",54 "video_id": "TWu4Ffb8LX3T",55 "video_url": "https://www.instagram.com/big.blue.sea/reel/DQ6wSS5gnTM",56 "video_scene": "000005_000007",57 "id": 0,58 "original_video_id": "DQ6wSS5gnTM",59 "turns": [60 {61 "turn_id": 1,62 "id": 0,63 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of stellate rabbitfish relative to coral\", \"ground_type\": \"anything\"}}}]",64 "user_query": "What is the relative spatial relationship of stellate rabbitfish with respect to coral? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",65 "visual_input": [66 "frames/frame_000000.jpg",67 "frames/frame_000001.jpg",68 "frames/frame_000002.jpg"69 ],70 "tool_output": "{\"boxes\": [[[\"247.0327606201172\", \"341.46514892578125\", \"722.6377563476562\", \"755.986328125\"], [\"3.3051037788391113\", \"358.26239013671875\", \"722.8549194335938\", \"1276.29052734375\"]], [[\"213.98171997070312\", \"330.00787353515625\", \"724.9449462890625\", \"738.626953125\"], [\"3.8461804389953613\", \"342.9699401855469\", \"723.3826904296875\", \"1276.8126220703125\"]], [[\"209.9815673828125\", \"365.73577880859375\", \"718.5889892578125\", \"730.7915649414062\"], [\"5.302748680114746\", \"321.976806640625\", \"724.8679809570312\", \"1277.0765380859375\"]]]}"71 },72 {73 "id": 0,74 "turn_id": 2,75 "user_query": "What is the relative spatial relationship of stellate rabbitfish with respect to coral? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",76 "assistant_response": "<answer>{\"x\": \"right\", \"y\": \"top\", \"z\": \"front\"}</answer>",77 "visual_input": [78 "frames/frame_000000.jpg",79 "frames/frame_000001.jpg",80 "frames/frame_000002.jpg"81 ]82 }83 ]84 },85 {86 "question": "What is the relative spatial relationship of shark with respect to coral? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",87 "answer": "{\"x\": \"left\", \"y\": \"top\", \"z\": \"behind\"}",88 "explanation": {89 "0": {90 "shark": [91 "-2.013920545578003",92 "77.67330169677734",93 "351.18316650390625",94 "455.9643859863281"95 ],96 "coral": [97 "5.42675256729126",98 "767.1176147460938",99 "724.9501953125",100 "1276.419189453125"101 ]102 },103 "1": {104 "shark": [105 "-3.6712896823883057",106 "71.15737915039062",107 "358.8427429199219",108 "421.6842956542969"109 ],110 "coral": [111 "1.9726037979125977",112 "768.3916625976562",113 "721.4632568359375",114 "1278.3314208984375"115 ]116 },117 "2": {118 "shark": [119 "-2.53908634185791",120 "74.14871215820312",121 "618.7841186523438",122 "427.885986328125"123 ],124 "coral": [125 "1.578383445739746",126 "770.8416137695312",127 "721.0828247070312",128 "1277.1947021484375"129 ]130 }131 },132 "dimension": "SpatialReasoning",133 "subdimension": "SpatialRelationAssessment",134 "question_task": "VideoQuestionAnswering",135 "question_format": "ExactMatchQuestion",136 "video_id": "dh0bhBqxooFx",137 "video_url": "https://www.instagram.com/dive_down_under/reel/DHMWNUczz1D",138 "video_scene": "000008_000011",139 "id": 1,140 "original_video_id": "DHMWNUczz1D",141 "turns": [142 {143 "turn_id": 1,144 "id": 1,145 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of shark with respect to coral\", \"ground_type\": \"anything\"}}}]",146 "user_query": "What is the relative spatial relationship of shark with respect to coral? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",147 "visual_input": [148 "frames/frame_000000.jpg",149 "frames/frame_000001.jpg",150 "frames/frame_000002.jpg"151 ],152 "tool_output": "{\"boxes\": [[[\"-2.013920545578003\", \"77.67330169677734\", \"351.18316650390625\", \"455.9643859863281\"], [\"5.42675256729126\", \"767.1176147460938\", \"724.9501953125\", \"1276.419189453125\"]], [[\"-3.6712896823883057\", \"71.15737915039062\", \"358.8427429199219\", \"421.6842956542969\"], [\"1.9726037979125977\", \"768.3916625976562\", \"721.4632568359375\", \"1278.3314208984375\"]], [[\"-2.53908634185791\", \"74.14871215820312\", \"618.7841186523438\", \"427.885986328125\"], [\"1.578383445739746\", \"770.8416137695312\", \"721.0828247070312\", \"1277.1947021484375\"]]]}"153 },154 {155 "id": 1,156 "turn_id": 2,157 "user_query": "What is the relative spatial relationship of shark with respect to coral? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",158 "assistant_response": "<answer>{\"x\": \"left\", \"y\": \"top\", \"z\": \"behind\"}</answer>",159 "visual_input": [160 "frames/frame_000000.jpg",161 "frames/frame_000001.jpg",162 "frames/frame_000002.jpg"163 ]164 }165 ]166 },167 {168 "question": "What is the relative spatial relationship of orange filefish with respect to grouper (steve)? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",169 "answer": "{\"x\": \"left\", \"y\": \"bottom\", \"z\": \"front\"}",170 "explanation": {171 "0": {172 "orange filefish": [173 "42.07930374145508",174 "698.9535522460938",175 "99.15799713134766",176 "779.5858764648438"177 ],178 "grouper (steve)": [179 "305.53472900390625",180 "516.1365356445312",181 "721.8081665039062",182 "710.5230102539062"183 ]184 },185 "1": {186 "orange filefish": [187 "64.93749237060547",188 "695.809814453125",189 "115.26131439208984",190 "767.732666015625"191 ],192 "grouper (steve)": [193 "291.4222106933594",194 "506.8061828613281",195 "725.6942749023438",196 "705.3982543945312"197 ]198 },199 "2": {200 "orange filefish": [201 "117.7239990234375",202 "691.5300903320312",203 "195.38377380371094",204 "828.1485595703125"205 ],206 "grouper (steve)": [207 "286.7666015625",208 "541.0834350585938",209 "722.8221435546875",210 "737.6067504882812"211 ]212 }213 },214 "dimension": "SpatialReasoning",215 "subdimension": "SpatialRelationAssessment",216 "question_task": "VideoQuestionAnswering",217 "question_format": "ExactMatchQuestion",218 "video_id": "rWTXrVAEqjhV",219 "video_url": "https://www.instagram.com/big.blue.sea/reel/DPzGMVQjG2R",220 "video_scene": "000023_000029",221 "id": 2,222 "original_video_id": "DPzGMVQjG2R",223 "turns": [224 {225 "turn_id": 1,226 "id": 2,227 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of orange filefish with respect to grouper (steve)\", \"ground_type\": \"anything\"}}}]",228 "user_query": "What is the relative spatial relationship of orange filefish with respect to grouper (steve)? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",229 "visual_input": [230 "frames/frame_000000.jpg",231 "frames/frame_000001.jpg",232 "frames/frame_000002.jpg"233 ],234 "tool_output": "{\"boxes\": [[[\"42.07930374145508\", \"698.9535522460938\", \"99.15799713134766\", \"779.5858764648438\"], [\"305.53472900390625\", \"516.1365356445312\", \"721.8081665039062\", \"710.5230102539062\"]], [[\"64.93749237060547\", \"695.809814453125\", \"115.26131439208984\", \"767.732666015625\"], [\"291.4222106933594\", \"506.8061828613281\", \"725.6942749023438\", \"705.3982543945312\"]], [[\"117.7239990234375\", \"691.5300903320312\", \"195.38377380371094\", \"828.1485595703125\"], [\"286.7666015625\", \"541.0834350585938\", \"722.8221435546875\", \"737.6067504882812\"]]]}"235 },236 {237 "id": 2,238 "turn_id": 2,239 "user_query": "What is the relative spatial relationship of orange filefish with respect to grouper (steve)? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",240 "assistant_response": "<answer>{\"x\": \"left\", \"y\": \"bottom\", \"z\": \"front\"}</answer>",241 "visual_input": [242 "frames/frame_000000.jpg",243 "frames/frame_000001.jpg",244 "frames/frame_000002.jpg"245 ]246 }247 ]248 },249 {250 "question": "What is the relative spatial relationship of sponge with respect to cuttlefish? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",251 "answer": "{\"x\": \"left\", \"y\": \"bottom\", \"z\": \"front\"}",252 "explanation": {253 "0": {254 "sponge": [255 "-2.126905918121338",256 "564.9112548828125",257 "717.37548828125",258 "1276.6123046875"259 ],260 "cuttlefish": [261 "155.8807830810547",262 "574.973388671875",263 "728.889892578125",264 "1005.9937133789062"265 ]266 },267 "1": {268 "sponge": [269 "-0.2795290946960449",270 "557.1808471679688",271 "718.5989990234375",272 "1279.739990234375"273 ],274 "cuttlefish": [275 "2.940795421600342",276 "611.8753051757812",277 "722.3512573242188",278 "1095.270263671875"279 ]280 },281 "2": {282 "sponge": [283 "2.5069212913513184",284 "554.4945068359375",285 "721.5560302734375",286 "1278.2781982421875"287 ],288 "cuttlefish": [289 "5.063302516937256",290 "640.8720703125",291 "724.6019287109375",292 "1052.793212890625"293 ]294 },295 "3": {296 "sponge": [297 "2.5011062622070312",298 "546.353271484375",299 "721.8387451171875",300 "1278.95703125"301 ],302 "cuttlefish": [303 "4.781970977783203",304 "618.0071411132812",305 "724.295654296875",306 "1025.27734375"307 ]308 },309 "4": {310 "sponge": [311 "-1.7758798599243164",312 "547.12353515625",313 "717.492431640625",314 "1279.2454833984375"315 ],316 "cuttlefish": [317 "1.1920809745788574",318 "597.1430053710938",319 "720.6553344726562",320 "1005.1435546875"321 ]322 },323 "5": {324 "sponge": [325 "0.706944465637207",326 "535.611572265625",327 "720.2102661132812",328 "1276.3905029296875"329 ],330 "cuttlefish": [331 "2.5107407569885254",332 "499.67919921875",333 "721.935302734375",334 "957.0596923828125"335 ]336 },337 "6": {338 "sponge": [339 "2.3313546180725098",340 "498.4603271484375",341 "721.8511962890625",342 "1274.4901123046875"343 ],344 "cuttlefish": [345 "4.44122314453125",346 "455.2911376953125",347 "723.8876342773438",348 "974.860107421875"349 ]350 },351 "7": {352 "sponge": [353 "0.30448436737060547",354 "576.951416015625",355 "719.7024536132812",356 "1277.94189453125"357 ],358 "cuttlefish": [359 "4.0343427658081055",360 "471.23193359375",361 "723.4749145507812",362 "975.5133666992188"363 ]364 }365 },366 "dimension": "SpatialReasoning",367 "subdimension": "SpatialRelationAssessment",368 "question_task": "VideoQuestionAnswering",369 "question_format": "ExactMatchQuestion",370 "video_id": "IpLypVL7GRn6",371 "video_url": "https://www.instagram.com/dive_down_under/reel/C1Af4x1hIbS",372 "video_scene": "000022_000030",373 "id": 3,374 "original_video_id": "C1Af4x1hIbS",375 "turns": [376 {377 "turn_id": 1,378 "id": 3,379 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of sponge with respect to cuttlefish\", \"ground_type\": \"anything\"}}}]",380 "user_query": "What is the relative spatial relationship of sponge with respect to cuttlefish? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",381 "visual_input": [382 "frames/frame_000000.jpg",383 "frames/frame_000001.jpg",384 "frames/frame_000002.jpg",385 "frames/frame_000003.jpg",386 "frames/frame_000004.jpg",387 "frames/frame_000005.jpg",388 "frames/frame_000006.jpg",389 "frames/frame_000007.jpg"390 ],391 "tool_output": "{\"boxes\": [[[\"-2.126905918121338\", \"564.9112548828125\", \"717.37548828125\", \"1276.6123046875\"], [\"155.8807830810547\", \"574.973388671875\", \"728.889892578125\", \"1005.9937133789062\"]], [[\"-0.2795290946960449\", \"557.1808471679688\", \"718.5989990234375\", \"1279.739990234375\"], [\"2.940795421600342\", \"611.8753051757812\", \"722.3512573242188\", \"1095.270263671875\"]], [[\"2.5069212913513184\", \"554.4945068359375\", \"721.5560302734375\", \"1278.2781982421875\"], [\"5.063302516937256\", \"640.8720703125\", \"724.6019287109375\", \"1052.793212890625\"]], [[\"2.5011062622070312\", \"546.353271484375\", \"721.8387451171875\", \"1278.95703125\"], [\"4.781970977783203\", \"618.0071411132812\", \"724.295654296875\", \"1025.27734375\"]], [[\"-1.7758798599243164\", \"547.12353515625\", \"717.492431640625\", \"1279.2454833984375\"], [\"1.1920809745788574\", \"597.1430053710938\", \"720.6553344726562\", \"1005.1435546875\"]], [[\"0.706944465637207\", \"535.611572265625\", \"720.2102661132812\", \"1276.3905029296875\"], [\"2.5107407569885254\", \"499.67919921875\", \"721.935302734375\", \"957.0596923828125\"]], [[\"2.3313546180725098\", \"498.4603271484375\", \"721.8511962890625\", \"1274.4901123046875\"], [\"4.44122314453125\", \"455.2911376953125\", \"723.8876342773438\", \"974.860107421875\"]], [[\"0.30448436737060547\", \"576.951416015625\", \"719.7024536132812\", \"1277.94189453125\"], [\"4.0343427658081055\", \"471.23193359375\", \"723.4749145507812\", \"975.5133666992188\"]]]}"392 },393 {394 "id": 3,395 "turn_id": 2,396 "user_query": "What is the relative spatial relationship of sponge with respect to cuttlefish? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",397 "assistant_response": "<answer>{\"x\": \"left\", \"y\": \"bottom\", \"z\": \"front\"}</answer>",398 "visual_input": [399 "frames/frame_000000.jpg",400 "frames/frame_000001.jpg",401 "frames/frame_000002.jpg",402 "frames/frame_000003.jpg",403 "frames/frame_000004.jpg",404 "frames/frame_000005.jpg",405 "frames/frame_000006.jpg",406 "frames/frame_000007.jpg"407 ]408 }409 ]410 },411 {412 "question": "What is the relative spatial relationship of giant clam with respect to sea cucumber? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",413 "answer": "{\"x\": \"right\", \"y\": \"bottom\", \"z\": \"front\"}",414 "explanation": {415 "0": {416 "giant clam": [417 "233.98886108398438",418 "522.2757568359375",419 "669.205078125",420 "1245.6824951171875"421 ],422 "sea cucumber": [423 "36.335777282714844",424 "513.1492309570312",425 "669.8904418945312",426 "1247.382568359375"427 ]428 },429 "1": {430 "giant clam": [431 "230.8139190673828",432 "519.226806640625",433 "671.5902099609375",434 "1245.6090087890625"435 ],436 "sea cucumber": [437 "38.164485931396484",438 "510.671630859375",439 "671.8484497070312",440 "1247.1842041015625"441 ]442 },443 "2": {444 "giant clam": [445 "228.73123168945312",446 "520.6299438476562",447 "670.5955200195312",448 "1244.386474609375"449 ],450 "sea cucumber": [451 "36.11117935180664",452 "507.92254638671875",453 "670.7056884765625",454 "1245.7288818359375"455 ]456 }457 },458 "dimension": "SpatialReasoning",459 "subdimension": "SpatialRelationAssessment",460 "question_task": "VideoQuestionAnswering",461 "question_format": "ExactMatchQuestion",462 "video_id": "sMvXAEa3zXr3",463 "video_url": "https://www.instagram.com/dive_down_under/reel/DRc_wSqE300",464 "video_scene": "000000_000006",465 "id": 4,466 "original_video_id": "DRc_wSqE300",467 "turns": [468 {469 "turn_id": 1,470 "id": 4,471 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of giant clam with respect to sea cucumber\", \"ground_type\": \"anything\"}}}]",472 "user_query": "What is the relative spatial relationship of giant clam with respect to sea cucumber? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",473 "visual_input": [474 "frames/frame_000000.jpg",475 "frames/frame_000001.jpg",476 "frames/frame_000002.jpg"477 ],478 "tool_output": "{\"boxes\": [[[\"233.98886108398438\", \"522.2757568359375\", \"669.205078125\", \"1245.6824951171875\"], [\"36.335777282714844\", \"513.1492309570312\", \"669.8904418945312\", \"1247.382568359375\"]], [[\"230.8139190673828\", \"519.226806640625\", \"671.5902099609375\", \"1245.6090087890625\"], [\"38.164485931396484\", \"510.671630859375\", \"671.8484497070312\", \"1247.1842041015625\"]], [[\"228.73123168945312\", \"520.6299438476562\", \"670.5955200195312\", \"1244.386474609375\"], [\"36.11117935180664\", \"507.92254638671875\", \"670.7056884765625\", \"1245.7288818359375\"]]]}"479 },480 {481 "id": 4,482 "turn_id": 2,483 "user_query": "What is the relative spatial relationship of giant clam with respect to sea cucumber? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",484 "assistant_response": "<answer>{\"x\": \"right\", \"y\": \"bottom\", \"z\": \"front\"}</answer>",485 "visual_input": [486 "frames/frame_000000.jpg",487 "frames/frame_000001.jpg",488 "frames/frame_000002.jpg"489 ]490 }491 ]492 },493 {494 "question": "What is the relative spatial relationship of green sea turtle with respect to sea urchin? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",495 "answer": "{\"x\": \"right\", \"y\": \"top\", \"z\": \"behind\"}",496 "explanation": {497 "0": {498 "green sea turtle": [499 "46.366703033447266",500 "669.2630004882812",501 "723.4484252929688",502 "1000.65380859375"503 ],504 "sea urchin": [505 "0.5638217926025391",506 "858.240478515625",507 "718.3004760742188",508 "1279.8109130859375"509 ]510 },511 "1": {512 "green sea turtle": [513 "51.27617645263672",514 "677.9867553710938",515 "724.119873046875",516 "1001.4251708984375"517 ],518 "sea urchin": [519 "1.6331219673156738",520 "860.7138671875",521 "720.4041137695312",522 "1278.9124755859375"523 ]524 },525 "2": {526 "green sea turtle": [527 "49.65281677246094",528 "677.545166015625",529 "722.4050903320312",530 "1001.737060546875"531 ],532 "sea urchin": [533 "0.7734417915344238",534 "854.8172607421875",535 "719.01123046875",536 "1279.2152099609375"537 ]538 },539 "3": {540 "green sea turtle": [541 "29.780073165893555",542 "667.27587890625",543 "722.8404541015625",544 "987.85595703125"545 ],546 "sea urchin": [547 "1.6520476341247559",548 "856.17236328125",549 "719.7572021484375",550 "1279.4521484375"551 ]552 },553 "4": {554 "green sea turtle": [555 "24.60793685913086",556 "663.7122192382812",557 "724.5050048828125",558 "984.6053466796875"559 ],560 "sea urchin": [561 "2.3447442054748535",562 "858.3611450195312",563 "720.6130981445312",564 "1279.9052734375"565 ]566 },567 "5": {568 "green sea turtle": [569 "50.0669059753418",570 "674.2950439453125",571 "724.0882568359375",572 "1003.758056640625"573 ],574 "sea urchin": [575 "1.6354823112487793",576 "860.313232421875",577 "720.480224609375",578 "1278.411865234375"579 ]580 },581 "6": {582 "green sea turtle": [583 "49.536495208740234",584 "671.3843994140625",585 "721.6973876953125",586 "1001.74609375"587 ],588 "sea urchin": [589 "0.20788192749023438",590 "862.5316162109375",591 "718.7535400390625",592 "1279.0186767578125"593 ]594 },595 "7": {596 "green sea turtle": [597 "37.055789947509766",598 "666.1902465820312",599 "723.7777709960938",600 "997.8011474609375"601 ],602 "sea urchin": [603 "1.6739988327026367",604 "862.0101928710938",605 "720.0487670898438",606 "1279.38037109375"607 ]608 },609 "8": {610 "green sea turtle": [611 "30.21057891845703",612 "680.394287109375",613 "724.2379150390625",614 "999.9774169921875"615 ],616 "sea urchin": [617 "2.0186305046081543",618 "860.413818359375",619 "720.6875610351562",620 "1278.64208984375"621 ]622 },623 "9": {624 "green sea turtle": [625 "28.293657302856445",626 "675.3681640625",627 "721.9268798828125",628 "1002.9205322265625"629 ],630 "sea urchin": [631 "0.4694938659667969",632 "857.408935546875",633 "719.0883178710938",634 "1278.6337890625"635 ]636 },637 "10": {638 "green sea turtle": [639 "52.785743713378906",640 "667.16748046875",641 "722.3040161132812",642 "1003.1057739257812"643 ],644 "sea urchin": [645 "1.6275858879089355",646 "861.0047607421875",647 "720.1951904296875",648 "1279.170166015625"649 ]650 },651 "11": {652 "green sea turtle": [653 "30.49869155883789",654 "655.1051025390625",655 "723.07666015625",656 "992.1971435546875"657 ],658 "sea urchin": [659 "1.4713525772094727",660 "859.1943969726562",661 "720.139404296875",662 "1278.7218017578125"663 ]664 },665 "12": {666 "green sea turtle": [667 "65.5318603515625",668 "676.1290283203125",669 "722.9884643554688",670 "1006.0244750976562"671 ],672 "sea urchin": [673 "0.9212207794189453",674 "856.585693359375",675 "718.732177734375",676 "1279.3759765625"677 ]678 },679 "13": {680 "green sea turtle": [681 "62.54752731323242",682 "675.9776611328125",683 "722.371337890625",684 "1006.8702392578125"685 ],686 "sea urchin": [687 "0.5432224273681641",688 "858.084228515625",689 "718.88525390625",690 "1279.192138671875"691 ]692 }693 },694 "dimension": "SpatialReasoning",695 "subdimension": "SpatialRelationAssessment",696 "question_task": "VideoQuestionAnswering",697 "question_format": "ExactMatchQuestion",698 "video_id": "pkdQ563LfsYt",699 "video_url": "https://www.instagram.com/dive_down_under/reel/C4EN_hBhEBB",700 "video_scene": "000006_000030",701 "id": 5,702 "original_video_id": "C4EN_hBhEBB",703 "turns": [704 {705 "turn_id": 1,706 "id": 5,707 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of green sea turtle with respect to sea urchin\", \"ground_type\": \"anything\"}}}]",708 "user_query": "What is the relative spatial relationship of green sea turtle with respect to sea urchin? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",709 "visual_input": [710 "frames/frame_000000.jpg",711 "frames/frame_000001.jpg",712 "frames/frame_000002.jpg",713 "frames/frame_000003.jpg",714 "frames/frame_000004.jpg",715 "frames/frame_000005.jpg",716 "frames/frame_000006.jpg",717 "frames/frame_000007.jpg",718 "frames/frame_000008.jpg",719 "frames/frame_000009.jpg",720 "frames/frame_000010.jpg",721 "frames/frame_000011.jpg",722 "frames/frame_000012.jpg",723 "frames/frame_000013.jpg"724 ],725 "tool_output": "{\"boxes\": [[[\"46.366703033447266\", \"669.2630004882812\", \"723.4484252929688\", \"1000.65380859375\"], [\"0.5638217926025391\", \"858.240478515625\", \"718.3004760742188\", \"1279.8109130859375\"]], [[\"51.27617645263672\", \"677.9867553710938\", \"724.119873046875\", \"1001.4251708984375\"], [\"1.6331219673156738\", \"860.7138671875\", \"720.4041137695312\", \"1278.9124755859375\"]], [[\"49.65281677246094\", \"677.545166015625\", \"722.4050903320312\", \"1001.737060546875\"], [\"0.7734417915344238\", \"854.8172607421875\", \"719.01123046875\", \"1279.2152099609375\"]], [[\"29.780073165893555\", \"667.27587890625\", \"722.8404541015625\", \"987.85595703125\"], [\"1.6520476341247559\", \"856.17236328125\", \"719.7572021484375\", \"1279.4521484375\"]], [[\"24.60793685913086\", \"663.7122192382812\", \"724.5050048828125\", \"984.6053466796875\"], [\"2.3447442054748535\", \"858.3611450195312\", \"720.6130981445312\", \"1279.9052734375\"]], [[\"50.0669059753418\", \"674.2950439453125\", \"724.0882568359375\", \"1003.758056640625\"], [\"1.6354823112487793\", \"860.313232421875\", \"720.480224609375\", \"1278.411865234375\"]], [[\"49.536495208740234\", \"671.3843994140625\", \"721.6973876953125\", \"1001.74609375\"], [\"0.20788192749023438\", \"862.5316162109375\", \"718.7535400390625\", \"1279.0186767578125\"]], [[\"37.055789947509766\", \"666.1902465820312\", \"723.7777709960938\", \"997.8011474609375\"], [\"1.6739988327026367\", \"862.0101928710938\", \"720.0487670898438\", \"1279.38037109375\"]], [[\"30.21057891845703\", \"680.394287109375\", \"724.2379150390625\", \"999.9774169921875\"], [\"2.0186305046081543\", \"860.413818359375\", \"720.6875610351562\", \"1278.64208984375\"]], [[\"28.293657302856445\", \"675.3681640625\", \"721.9268798828125\", \"1002.9205322265625\"], [\"0.4694938659667969\", \"857.408935546875\", \"719.0883178710938\", \"1278.6337890625\"]], [[\"52.785743713378906\", \"667.16748046875\", \"722.3040161132812\", \"1003.1057739257812\"], [\"1.6275858879089355\", \"861.0047607421875\", \"720.1951904296875\", \"1279.170166015625\"]], [[\"30.49869155883789\", \"655.1051025390625\", \"723.07666015625\", \"992.1971435546875\"], [\"1.4713525772094727\", \"859.1943969726562\", \"720.139404296875\", \"1278.7218017578125\"]], [[\"65.5318603515625\", \"676.1290283203125\", \"722.9884643554688\", \"1006.0244750976562\"], [\"0.9212207794189453\", \"856.585693359375\", \"718.732177734375\", \"1279.3759765625\"]], [[\"62.54752731323242\", \"675.9776611328125\", \"722.371337890625\", \"1006.8702392578125\"], [\"0.5432224273681641\", \"858.084228515625\", \"718.88525390625\", \"1279.192138671875\"]]]}"726 },727 {728 "id": 5,729 "turn_id": 2,730 "user_query": "What is the relative spatial relationship of green sea turtle with respect to sea urchin? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",731 "assistant_response": "<answer>{\"x\": \"right\", \"y\": \"top\", \"z\": \"behind\"}</answer>",732 "visual_input": [733 "frames/frame_000000.jpg",734 "frames/frame_000001.jpg",735 "frames/frame_000002.jpg",736 "frames/frame_000003.jpg",737 "frames/frame_000004.jpg",738 "frames/frame_000005.jpg",739 "frames/frame_000006.jpg",740 "frames/frame_000007.jpg",741 "frames/frame_000008.jpg",742 "frames/frame_000009.jpg",743 "frames/frame_000010.jpg",744 "frames/frame_000011.jpg",745 "frames/frame_000012.jpg",746 "frames/frame_000013.jpg"747 ]748 }749 ]750 },751 {752 "question": "What is the relative spatial relationship of sponge with respect to crab? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",753 "answer": "{\"x\": \"right\", \"y\": \"top\", \"z\": \"front\"}",754 "explanation": {755 "0": {756 "sponge": [757 "0.7415556907653809",758 "189.47203063964844",759 "717.890625",760 "648.0792846679688"761 ],762 "crab": [763 "104.28826904296875",764 "602.84130859375",765 "336.9621887207031",766 "787.2913818359375"767 ]768 },769 "1": {770 "sponge": [771 "1.238107681274414",772 "189.03805541992188",773 "718.8942260742188",774 "592.9767456054688"775 ],776 "crab": [777 "164.3001251220703",778 "559.7354736328125",779 "337.6840515136719",780 "719.3505859375"781 ]782 },783 "2": {784 "sponge": [785 "4.048504829406738",786 "189.13674926757812",787 "720.1910400390625",788 "550.013916015625"789 ],790 "crab": [791 "189.83267211914062",792 "473.66326904296875",793 "380.5851745605469",794 "650.6090087890625"795 ]796 }797 },798 "dimension": "SpatialReasoning",799 "subdimension": "SpatialRelationAssessment",800 "question_task": "VideoQuestionAnswering",801 "question_format": "ExactMatchQuestion",802 "video_id": "A3RX6ClI2Swg",803 "video_url": "https://www.instagram.com/onebreathdiver/reel/DC7kwvPTTSy",804 "video_scene": "000019_000022",805 "id": 6,806 "original_video_id": "DC7kwvPTTSy",807 "turns": [808 {809 "turn_id": 1,810 "id": 6,811 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of sponge with respect to crab\", \"ground_type\": \"anything\"}}}]",812 "user_query": "What is the relative spatial relationship of sponge with respect to crab? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",813 "visual_input": [814 "frames/frame_000000.jpg",815 "frames/frame_000001.jpg",816 "frames/frame_000002.jpg"817 ],818 "tool_output": "{\"boxes\": [[[\"0.7415556907653809\", \"189.47203063964844\", \"717.890625\", \"648.0792846679688\"], [\"104.28826904296875\", \"602.84130859375\", \"336.9621887207031\", \"787.2913818359375\"]], [[\"1.238107681274414\", \"189.03805541992188\", \"718.8942260742188\", \"592.9767456054688\"], [\"164.3001251220703\", \"559.7354736328125\", \"337.6840515136719\", \"719.3505859375\"]], [[\"4.048504829406738\", \"189.13674926757812\", \"720.1910400390625\", \"550.013916015625\"], [\"189.83267211914062\", \"473.66326904296875\", \"380.5851745605469\", \"650.6090087890625\"]]]}"819 },820 {821 "id": 6,822 "turn_id": 2,823 "user_query": "What is the relative spatial relationship of sponge with respect to crab? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",824 "assistant_response": "<answer>{\"x\": \"right\", \"y\": \"top\", \"z\": \"front\"}</answer>",825 "visual_input": [826 "frames/frame_000000.jpg",827 "frames/frame_000001.jpg",828 "frames/frame_000002.jpg"829 ]830 }831 ]832 },833 {834 "question": "What is the relative spatial relationship of white-spotted puffer fish with respect to coral reef structure? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",835 "answer": "{\"x\": \"right\", \"y\": \"bottom\", \"z\": \"front\"}",836 "explanation": {837 "0": {838 "white-spotted puffer fish": [839 "349.8952331542969",840 "812.8986206054688",841 "721.5775756835938",842 "967.1619262695312"843 ],844 "coral reef structure": [845 "193.5162353515625",846 "0.1946258544921875",847 "720.0460205078125",848 "873.6254272460938"849 ]850 },851 "1": {852 "white-spotted puffer fish": [853 "313.9132385253906",854 "789.5278930664062",855 "722.0670776367188",856 "976.176025390625"857 ],858 "coral reef structure": [859 "162.2484130859375",860 "-2.2369003295898438",861 "720.845947265625",862 "832.484130859375"863 ]864 },865 "2": {866 "white-spotted puffer fish": [867 "273.8776550292969",868 "769.103271484375",869 "722.1289672851562",870 "970.7489013671875"871 ],872 "coral reef structure": [873 "136.68377685546875",874 "0.5384445190429688",875 "720.1586303710938",876 "807.859619140625"877 ]878 }879 },880 "dimension": "SpatialReasoning",881 "subdimension": "SpatialRelationAssessment",882 "question_task": "VideoQuestionAnswering",883 "question_format": "ExactMatchQuestion",884 "video_id": "S5zkFd4Nve6C",885 "video_url": "https://www.instagram.com/big.blue.sea/reel/C9ugKmtNxNW",886 "video_scene": "000000_000003",887 "id": 7,888 "original_video_id": "C9ugKmtNxNW",889 "turns": [890 {891 "turn_id": 1,892 "id": 7,893 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of white-spotted puffer fish relative to coral reef structure\", \"ground_type\": \"anything\"}}}]",894 "user_query": "What is the relative spatial relationship of white-spotted puffer fish with respect to coral reef structure? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",895 "visual_input": [896 "frames/frame_000000.jpg",897 "frames/frame_000001.jpg",898 "frames/frame_000002.jpg"899 ],900 "tool_output": "{\"boxes\": [[[\"349.8952331542969\", \"812.8986206054688\", \"721.5775756835938\", \"967.1619262695312\"], [\"193.5162353515625\", \"0.1946258544921875\", \"720.0460205078125\", \"873.6254272460938\"]], [[\"313.9132385253906\", \"789.5278930664062\", \"722.0670776367188\", \"976.176025390625\"], [\"162.2484130859375\", \"-2.2369003295898438\", \"720.845947265625\", \"832.484130859375\"]], [[\"273.8776550292969\", \"769.103271484375\", \"722.1289672851562\", \"970.7489013671875\"], [\"136.68377685546875\", \"0.5384445190429688\", \"720.1586303710938\", \"807.859619140625\"]]]}"901 },902 {903 "id": 7,904 "turn_id": 2,905 "user_query": "What is the relative spatial relationship of white-spotted puffer fish with respect to coral reef structure? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",906 "assistant_response": "<answer>{\"x\": \"right\", \"y\": \"bottom\", \"z\": \"front\"}</answer>",907 "visual_input": [908 "frames/frame_000000.jpg",909 "frames/frame_000001.jpg",910 "frames/frame_000002.jpg"911 ]912 }913 ]914 },915 {916 "question": "What is the relative spatial relationship of octopus with respect to bivalve shell (likely oyster or scallop)? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",917 "answer": "{\"x\": \"left\", \"y\": \"bottom\", \"z\": \"behind\"}",918 "explanation": {919 "0": {920 "octopus": [921 "340.9879150390625",922 "264.7485046386719",923 "733.3424072265625",924 "568.5324096679688"925 ],926 "bivalve shell (likely oyster or scallop)": [927 "427.3802185058594",928 "139.5309600830078",929 "921.8291015625",930 "459.21246337890625"931 ]932 },933 "1": {934 "octopus": [935 "439.2154235839844",936 "365.4620361328125",937 "707.39794921875",938 "547.7110595703125"939 ],940 "bivalve shell (likely oyster or scallop)": [941 "417.44232177734375",942 "138.99366760253906",943 "925.1295166015625",944 "464.33685302734375"945 ]946 },947 "2": {948 "octopus": [949 "438.0497741699219",950 "382.5807800292969",951 "748.6229858398438",952 "546.6159057617188"953 ],954 "bivalve shell (likely oyster or scallop)": [955 "430.70367431640625",956 "144.10316467285156",957 "940.97802734375",958 "471.5815734863281"959 ]960 },961 "3": {962 "octopus": [963 "477.7276306152344",964 "348.19732666015625",965 "758.543212890625",966 "498.4759826660156"967 ],968 "bivalve shell (likely oyster or scallop)": [969 "473.61505126953125",970 "109.73469543457031",971 "1002.7350463867188",972 "448.3147277832031"973 ]974 },975 "4": {976 "octopus": [977 "504.7557373046875",978 "319.8270568847656",979 "767.9239501953125",980 "496.9566955566406"981 ],982 "bivalve shell (likely oyster or scallop)": [983 "489.12176513671875",984 "71.52242279052734",985 "1038.523193359375",986 "422.3961181640625"987 ]988 },989 "5": {990 "octopus": [991 "509.3495178222656",992 "288.96356201171875",993 "786.88671875",994 "496.1146545410156"995 ],996 "bivalve shell (likely oyster or scallop)": [997 "486.14959716796875",998 "59.6580810546875",999 "1052.2303466796875",1000 "411.6819763183594"1001 ]1002 },1003 "6": {1004 "octopus": [1005 "506.3243408203125",1006 "280.40435791015625",1007 "817.0297241210938",1008 "506.0248107910156"1009 ],1010 "bivalve shell (likely oyster or scallop)": [1011 "463.6659240722656",1012 "30.063636779785156",1013 "1068.0518798828125",1014 "411.76104736328125"1015 ]1016 },1017 "7": {1018 "octopus": [1019 "492.45477294921875",1020 "302.9705810546875",1021 "839.7601928710938",1022 "492.7509765625"1023 ],1024 "bivalve shell (likely oyster or scallop)": [1025 "514.5900268554688",1026 "58.598411560058594",1027 "1010.789794921875",1028 "423.3692321777344"1029 ]1030 },1031 "8": {1032 "octopus": [1033 "499.896240234375",1034 "289.5732116699219",1035 "868.8851928710938",1036 "504.7474060058594"1037 ],1038 "bivalve shell (likely oyster or scallop)": [1039 "498.71923828125",1040 "81.96234130859375",1041 "969.8833618164062",1042 "450.80377197265625"1043 ]1044 }1045 },1046 "dimension": "SpatialReasoning",1047 "subdimension": "SpatialRelationAssessment",1048 "question_task": "VideoQuestionAnswering",1049 "question_format": "ExactMatchQuestion",1050 "video_id": "agpcfTWcOWm7",1051 "video_url": "https://www.instagram.com/onebreathdiver/reel/CKZrODUBs8V",1052 "video_scene": "000074_000104",1053 "id": 8,1054 "original_video_id": "CKZrODUBs8V",1055 "turns": [1056 {1057 "turn_id": 1,1058 "id": 8,1059 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of octopus with respect to bivalve shell\", \"ground_type\": \"anything\"}}}]",1060 "user_query": "What is the relative spatial relationship of octopus with respect to bivalve shell (likely oyster or scallop)? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",1061 "visual_input": [1062 "frames/frame_000000.jpg",1063 "frames/frame_000001.jpg",1064 "frames/frame_000002.jpg",1065 "frames/frame_000003.jpg",1066 "frames/frame_000004.jpg",1067 "frames/frame_000005.jpg",1068 "frames/frame_000006.jpg",1069 "frames/frame_000007.jpg",1070 "frames/frame_000008.jpg"1071 ],1072 "tool_output": "{\"boxes\": [[[\"340.9879150390625\", \"264.7485046386719\", \"733.3424072265625\", \"568.5324096679688\"], [\"427.3802185058594\", \"139.5309600830078\", \"921.8291015625\", \"459.21246337890625\"]], [[\"439.2154235839844\", \"365.4620361328125\", \"707.39794921875\", \"547.7110595703125\"], [\"417.44232177734375\", \"138.99366760253906\", \"925.1295166015625\", \"464.33685302734375\"]], [[\"438.0497741699219\", \"382.5807800292969\", \"748.6229858398438\", \"546.6159057617188\"], [\"430.70367431640625\", \"144.10316467285156\", \"940.97802734375\", \"471.5815734863281\"]], [[\"477.7276306152344\", \"348.19732666015625\", \"758.543212890625\", \"498.4759826660156\"], [\"473.61505126953125\", \"109.73469543457031\", \"1002.7350463867188\", \"448.3147277832031\"]], [[\"504.7557373046875\", \"319.8270568847656\", \"767.9239501953125\", \"496.9566955566406\"], [\"489.12176513671875\", \"71.52242279052734\", \"1038.523193359375\", \"422.3961181640625\"]], [[\"509.3495178222656\", \"288.96356201171875\", \"786.88671875\", \"496.1146545410156\"], [\"486.14959716796875\", \"59.6580810546875\", \"1052.2303466796875\", \"411.6819763183594\"]], [[\"506.3243408203125\", \"280.40435791015625\", \"817.0297241210938\", \"506.0248107910156\"], [\"463.6659240722656\", \"30.063636779785156\", \"1068.0518798828125\", \"411.76104736328125\"]], [[\"492.45477294921875\", \"302.9705810546875\", \"839.7601928710938\", \"492.7509765625\"], [\"514.5900268554688\", \"58.598411560058594\", \"1010.789794921875\", \"423.3692321777344\"]], [[\"499.896240234375\", \"289.5732116699219\", \"868.8851928710938\", \"504.7474060058594\"], [\"498.71923828125\", \"81.96234130859375\", \"969.8833618164062\", \"450.80377197265625\"]]]}"1073 },1074 {1075 "id": 8,1076 "turn_id": 2,1077 "user_query": "What is the relative spatial relationship of octopus with respect to bivalve shell (likely oyster or scallop)? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",1078 "assistant_response": "<answer>{\"x\": \"left\", \"y\": \"bottom\", \"z\": \"behind\"}</answer>",1079 "visual_input": [1080 "frames/frame_000000.jpg",1081 "frames/frame_000001.jpg",1082 "frames/frame_000002.jpg",1083 "frames/frame_000003.jpg",1084 "frames/frame_000004.jpg",1085 "frames/frame_000005.jpg",1086 "frames/frame_000006.jpg",1087 "frames/frame_000007.jpg",1088 "frames/frame_000008.jpg"1089 ]1090 }1091 ]1092 },1093 {1094 "question": "What is the relative spatial relationship of bluespotted stingray with respect to black and yellow fish? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",1095 "answer": "{\"x\": \"right\", \"y\": \"bottom\", \"z\": \"front\"}",1096 "explanation": {1097 "0": {1098 "bluespotted stingray": [1099 "219.23239135742188",1100 "652.4553833007812",1101 "719.9999389648438",1102 "921.6375122070312"1103 ],1104 "black and yellow fish": [1105 "96.91966247558594",1106 "352.8218994140625",1107 "156.0782470703125",1108 "405.9788513183594"1109 ]1110 },1111 "1": {1112 "bluespotted stingray": [1113 "219.53211975097656",1114 "653.002197265625",1115 "520.4584350585938",1116 "929.0731811523438"1117 ],1118 "black and yellow fish": [1119 "180.37619018554688",1120 "367.7116394042969",1121 "211.13491821289062",1122 "420.291748046875"1123 ]1124 },1125 "2": {1126 "bluespotted stingray": [1127 "215.42140197753906",1128 "640.5311279296875",1129 "438.7418518066406",1130 "871.976806640625"1131 ],1132 "black and yellow fish": [1133 "219.81117248535156",1134 "386.3087463378906",1135 "257.70361328125",1136 "440.4817199707031"1137 ]1138 },1139 "3": {1140 "bluespotted stingray": [1141 "260.2196350097656",1142 "636.5774536132812",1143 "460.64373779296875",1144 "829.09130859375"1145 ],1146 "black and yellow fish": [1147 "295.0250244140625",1148 "399.9673156738281",1149 "333.36163330078125",1150 "452.51019287109375"1151 ]1152 }1153 },1154 "dimension": "SpatialReasoning",1155 "subdimension": "SpatialRelationAssessment",1156 "question_task": "VideoQuestionAnswering",1157 "question_format": "ExactMatchQuestion",1158 "video_id": "cAbWWrmrrLLO",1159 "video_url": "https://www.instagram.com/big.blue.sea/reel/DJtlO4JNgqI",1160 "video_scene": "000017_000020",1161 "id": 9,1162 "original_video_id": "DJtlO4JNgqI",1163 "turns": [1164 {1165 "turn_id": 1,1166 "id": 9,1167 "assistant_response": "[{\"function\": {\"name\": \"object_grounding\", \"arguments\": {\"prompt\": \"A region of bluespotted stingray relative to black and yellow fish\", \"ground_type\": \"anything\"}}}]",1168 "user_query": "What is the relative spatial relationship of bluespotted stingray with respect to black and yellow fish? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",1169 "visual_input": [1170 "frames/frame_000000.jpg",1171 "frames/frame_000001.jpg",1172 "frames/frame_000002.jpg",1173 "frames/frame_000003.jpg"1174 ],1175 "tool_output": "{\"boxes\": [[[\"219.23239135742188\", \"652.4553833007812\", \"719.9999389648438\", \"921.6375122070312\"], [\"96.91966247558594\", \"352.8218994140625\", \"156.0782470703125\", \"405.9788513183594\"]], [[\"219.53211975097656\", \"653.002197265625\", \"520.4584350585938\", \"929.0731811523438\"], [\"180.37619018554688\", \"367.7116394042969\", \"211.13491821289062\", \"420.291748046875\"]], [[\"215.42140197753906\", \"640.5311279296875\", \"438.7418518066406\", \"871.976806640625\"], [\"219.81117248535156\", \"386.3087463378906\", \"257.70361328125\", \"440.4817199707031\"]], [[\"260.2196350097656\", \"636.5774536132812\", \"460.64373779296875\", \"829.09130859375\"], [\"295.0250244140625\", \"399.9673156738281\", \"333.36163330078125\", \"452.51019287109375\"]]]}"1176 },1177 {1178 "id": 9,1179 "turn_id": 2,1180 "user_query": "What is the relative spatial relationship of bluespotted stingray with respect to black and yellow fish? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",1181 "assistant_response": "<answer>{\"x\": \"right\", \"y\": \"bottom\", \"z\": \"front\"}</answer>",1182 "visual_input": [1183 "frames/frame_000000.jpg",1184 "frames/frame_000001.jpg",1185 "frames/frame_000002.jpg",1186 "frames/frame_000003.jpg"1187 ]1188 }1189 ]1190 },1191 {1192 "question": "What is the relative spatial relationship of parrotfish with respect to octopus? Please describe the direction along each spatial axis using a dictionary with the following keys and allowed values: 'x' (left, right, none), 'y' (top, bottom, none), and 'z' (front, behind, none).",1193 "answer": "{\"x\": \"left\", \"y\": \"top\", \"z\": \"behind\"}",1194 "explanation": {1195 "0": {1196 "parrotfish": [1197 "21.99608612060547",1198 "502.0540771484375",1199 "83.3436050415039",1200 "761.8826904296875"