Telkwevr/Bench2Drive-VL-base
Bench2Drive-VL: Full-Stack Software for Closed-Loop Autonomous Driving with Vision Language Models Project Page | GitHub | Paper Bench2Drive-VL is a comprehensive closed-loop benchmark for Vision-Language Models in Autonomous Driving (VLM4AD). It extends the Bench2Drive benchmark by introducing closed-loop evaluation and the DriveCommenter expert model for automated annotation. This repository contains the natural language annotations for the Bench2Drive-Base1000 dataset. These… See the full description on the dataset page: https://huggingface.co/datasets/Telkwevr/Bench2Drive-VL-base.
19.9k
1{2 "key_object_infos": {3 "<c7573<CAM_FRONT,805.0,491.7>>": {4 "id": "7573",5 "Category": "Vehicle",6 "Status": null,7 "Visual_description": "navy car",8 "Detailed_description": "the navy car that is to the front of the ego vehicle",9 "2d_bbox": {10 "CAM_FRONT": [11 [12 768.0,13 465.414 ],15 [16 842.0,17 518.018 ]19 ]20 },21 "3d_bbox": {22 "CAM_FRONT": [23 [24 35.0,25 1.1,26 -0.527 ],28 [29 35.0,30 1.1,31 -1.932 ],33 [34 35.0,35 -0.9,36 -0.537 ],38 [39 35.0,40 -0.9,41 -1.942 ],43 [44 30.8,45 1.1,46 -0.447 ],48 [49 30.8,50 1.1,51 -1.852 ],53 [54 30.8,55 -0.9,56 -0.457 ],58 [59 30.8,60 -0.9,61 -1.862 ]63 ]64 },65 "is_role": false,66 "is_dangerous": false,67 "obj_dict": {68 "class": "vehicle",69 "state": "dynamic",70 "id": "7573",71 "location": [72 77.552001953125,73 277.5265808105469,74 152.407394409179775 ],76 "rotation": [77 0.7966732382774353,78 0.0014057940570637584,79 -139.521926879882880 ],81 "bbx_loc": [82 -0.00038130462053231895,83 -3.0153989882819587e-06,84 0.692649960517883385 ],86 "center": [87 77.55962371826172,88 277.5330810546875,89 153.099975585937590 ],91 "extent": [92 2.0906050205230713,93 0.9970585703849792,94 0.692648053169250595 ],96 "world_cord": [97 [98 76.62413787841797,99 275.42388916015625,100 153.82159423828125101 ],102 [103 76.60946655273438,104 275.4114074707031,105 152.43643188476562106 ],107 [108 75.32964324951172,109 276.94073486328125,110 153.82164001464844111 ],112 [113 75.31497192382812,114 276.9282531738281,115 152.4364776611328116 ],117 [118 79.80428314208984,119 278.13787841796875,120 153.76345825195312121 ],122 [123 79.78961181640625,124 278.12542724609375,125 152.3782958984375126 ],127 [128 78.5097885131836,129 279.65472412109375,130 153.7635040283203131 ],132 [133 78.4951171875,134 279.6422424316406,135 152.3783416748047136 ]137 ],138 "semantic_tags": [139 14140 ],141 "type_id": "vehicle.audi.tt",142 "color": "0,39,105",143 "base_type": "car",144 "num_points": 17,145 "distance": 33.6893342529586,146 "speed": 7.316433940443128,147 "brake": 1.0,148 "light_state": "Brake",149 "road_id": 221,150 "lane_id": -2,151 "section_id": 1,152 "world2vehicle": [153 [154 -0.7605808973312378,155 -0.6490942239761353,156 0.01390412263572216,157 237.00637817382812158 ],159 [160 0.6491567492485046,161 -0.7606546878814697,162 -2.453336310281884e-05,163 160.76222229003906164 ],165 [166 0.010592159815132618,167 0.009007295593619347,168 0.9999033212661743,169 -155.7138671875170 ],171 [172 0.0,173 0.0,174 0.0,175 1.0176 ]177 ],178 "position": [179 33.69074934758814,180 0.13859739803334037,181 -0.2458591462903712182 ],183 "yaw": 0.0013997574689456477,184 "color_rgb": null,185 "color_name": null,186 "lane_type": 2,187 "lane_type_str": "Driving",188 "is_in_junction": false,189 "junction_id": -1,190 "distance_to_junction": 228.25572204589844,191 "next_junction_id": 1111,192 "next_road_ids": [193 1117194 ],195 "next_next_road_ids": [196 1392197 ],198 "same_road_as_ego": true,199 "same_direction_as_ego": true,200 "lane_relative_to_ego": 0,201 "steer": -0.003417452441407246,202 "vehicle_cuts_in": false203 },204 "position": [205 33.69074934758814,206 0.13859739803334037,207 -0.2458591462903712208 ],209 "distance": 33.6893342529586,210 "speed": [211 33.69074934758814,212 0.13859739803334037,213 -0.2458591462903712214 ],215 "same_dir": true,216 "is_sign": false217 },218 "<c7537<CAM_FRONT,263.6,664.8>>": {219 "id": "7537",220 "Category": "Pedestrian",221 "Status": null,222 "Visual_description": "Pedestrian",223 "Detailed_description": "the pedestrian to the front of the ego vehicle",224 "2d_bbox": {225 "CAM_FRONT": [226 [227 175.1,228 370.6229 ],230 [231 352.0,232 958.9233 ]234 ]235 },236 "3d_bbox": {237 "CAM_FRONT": [238 [239 4.0,240 -2.0,241 0.2242 ],243 [244 4.0,245 -2.0,246 -1.6247 ],248 [249 3.7,250 -2.0,251 0.3252 ],253 [254 3.6,255 -2.0,256 -1.6257 ],258 [259 4.0,260 -1.6,261 0.2262 ],263 [264 4.0,265 -1.6,266 -1.6267 ],268 [269 3.7,270 -1.6,271 0.2272 ],273 [274 3.6,275 -1.6,276 -1.6277 ]278 ]279 },280 "is_role": false,281 "is_dangerous": false,282 "obj_dict": {283 "class": "walker",284 "id": "7537",285 "location": [286 98.46060180664062,287 297.8371887207031,288 152.99786376953125289 ],290 "rotation": [291 0.7073206901550293,292 -2.668245713266515e-08,293 130.44573974609375294 ],295 "bbx_loc": [296 0.0,297 0.0,298 0.0299 ],300 "center": [301 98.46060180664062,302 297.8371887207031,303 152.99786376953125304 ],305 "extent": [306 0.18767888844013214,307 0.18767888844013214,308 0.9300000071525574309 ],310 "world_cord": [311 [312 98.20347595214844,313 297.8495178222656,314 153.93011474609375315 ],316 [317 98.18858337402344,318 297.86700439453125,319 152.07025146484375320 ],321 [322 98.4891357421875,323 298.093017578125,324 153.93011474609375325 ],326 [327 98.47423553466797,328 298.1105041503906,329 152.07025146484375330 ],331 [332 98.44696807861328,333 297.5638732910156,334 153.92547607421875335 ],336 [337 98.43206787109375,338 297.58135986328125,339 152.06561279296875340 ],341 [342 98.73262023925781,343 297.807373046875,344 153.92547607421875345 ],346 [347 98.71772766113281,348 297.8248596191406,349 152.06561279296875350 ]351 ],352 "semantic_tags": [353 12354 ],355 "type_id": "walker.pedestrian.0017",356 "gender": "male",357 "age": "adult",358 "num_points": 138,359 "distance": 4.934370660645002,360 "speed": 4.888364192800251e-06,361 "bone": [362 [363 [364 98.4810562133789,365 297.8285217285156,366 153.1227264404297367 ],368 [369 98.4874496459961,370 297.8206787109375,371 153.22935485839844372 ]373 ],374 [375 [376 98.4810562133789,377 297.8285217285156,378 153.1227264404297379 ],380 [381 98.4283447265625,382 297.76739501953125,383 153.035400390625384 ]385 ],386 [387 [388 98.4810562133789,389 297.8285217285156,390 153.1227264404297391 ],392 [393 98.55254364013672,394 297.8653564453125,395 153.03515625396 ]397 ],398 [399 [400 98.4874496459961,401 297.8206787109375,402 153.22935485839844403 ],404 [405 98.50664520263672,406 297.7951354980469,407 153.3918914794922408 ]409 ],410 [411 [412 98.50664520263672,413 297.7951354980469,414 153.3918914794922415 ],416 [417 98.56781005859375,418 297.82818603515625,419 153.54115295410156420 ]421 ],422 [423 [424 98.50664520263672,425 297.7951354980469,426 153.3918914794922427 ],428 [429 98.51061248779297,430 297.78729248046875,431 153.5872344970703432 ]433 ],434 [435 [436 98.50664520263672,437 297.7951354980469,438 153.3918914794922439 ],440 [441 98.45415496826172,442 297.7464904785156,443 153.54017639160156444 ]445 ],446 [447 [448 98.56781005859375,449 297.82818603515625,450 153.54115295410156451 ],452 [453 98.66819763183594,454 297.87109375,455 153.53839111328125456 ]457 ],458 [459 [460 98.66819763183594,461 297.87109375,462 153.53839111328125463 ],464 [465 98.75485229492188,466 297.880126953125,467 153.27708435058594468 ]469 ],470 [471 [472 98.75485229492188,473 297.880126953125,474 153.27708435058594475 ],476 [477 98.75862121582031,478 297.9472351074219,479 153.01681518554688480 ]481 ],482 [483 [484 98.75862121582031,485 297.9472351074219,486 153.01681518554688487 ],488 [489 98.73968505859375,490 297.9660949707031,491 152.98748779296875492 ]493 ],494 [495 [496 98.75862121582031,497 297.9472351074219,498 153.01681518554688499 ],500 [501 98.7555160522461,502 297.9911804199219,503 152.94337463378906504 ]505 ],506 [507 [508 98.75862121582031,509 297.9472351074219,510 153.01681518554688511 ],512 [513 98.7803726196289,514 297.9810791015625,515 152.94590759277344516 ]517 ],518 [519 [520 98.75862121582031,521 297.9472351074219,522 153.01681518554688523 ],524 [525 98.7918930053711,526 297.96343994140625,527 152.94528198242188528 ]529 ],530 [531 [532 98.75862121582031,533 297.9472351074219,534 153.01681518554688535 ],536 [537 98.7982406616211,538 297.94171142578125,539 152.95008850097656540 ]541 ],542 [543 [544 98.73968505859375,545 297.9660949707031,546 152.98748779296875547 ],548 [549 98.70794677734375,550 297.96380615234375,551 152.96202087402344552 ]553 ],554 [555 [556 98.70794677734375,557 297.96380615234375,558 152.96202087402344559 ],560 [561 98.70120239257812,562 297.9626159667969,563 152.93299865722656564 ]565 ],566 [567 [568 98.70120239257812,569 297.9626159667969,570 152.93299865722656571 ],572 [573 98.69667053222656,574 297.9598388671875,575 152.90560913085938576 ]577 ],578 [579 [580 98.7555160522461,581 297.9911804199219,582 152.94337463378906583 ],584 [585 98.7463150024414,586 297.99102783203125,587 152.89781188964844588 ]589 ],590 [591 [592 98.7463150024414,593 297.99102783203125,594 152.89781188964844595 ],596 [597 98.73121643066406,598 297.9723815917969,599 152.8763427734375600 ]601 ],602 [603 [604 98.73121643066406,605 297.9723815917969,606 152.8763427734375607 ],608 [609 98.72016143798828,610 297.954833984375,611 152.87103271484375612 ]613 ],614 [615 [616 98.7803726196289,617 297.9810791015625,618 152.94590759277344619 ],620 [621 98.76873779296875,622 297.9734802246094,623 152.90069580078125624 ]625 ],626 [627 [628 98.76873779296875,629 297.9734802246094,630 152.90069580078125631 ],632 [633 98.75353240966797,634 297.9574279785156,635 152.88116455078125636 ]637 ],638 [639 [640 98.75353240966797,641 297.9574279785156,642 152.88116455078125643 ],644 [645 98.73689270019531,646 297.93621826171875,647 152.8712921142578648 ]649 ],650 [651 [652 98.7918930053711,653 297.96343994140625,654 152.94528198242188655 ],656 [657 98.78072357177734,658 297.9500732421875,659 152.90899658203125660 ]661 ],662 [663 [664 98.78072357177734,665 297.9500732421875,666 152.90899658203125667 ],668 [669 98.76375579833984,670 297.930908203125,671 152.8930206298828672 ]673 ],674 [675 [676 98.76375579833984,677 297.930908203125,678 152.8930206298828679 ],680 [681 98.74795532226562,682 297.911376953125,683 152.88833618164062684 ]685 ],686 [687 [688 98.7982406616211,689 297.94171142578125,690 152.95008850097656691 ],692 [693 98.78829193115234,694 297.92962646484375,695 152.92608642578125696 ]697 ],698 [699 [700 98.78829193115234,701 297.92962646484375,702 152.92608642578125703 ],704 [705 98.77603912353516,706 297.9152526855469,707 152.9132080078125708 ]709 ],710 [711 [712 98.77603912353516,713 297.9152526855469,714 152.9132080078125715 ],716 [717 98.7647476196289,718 297.9030456542969,719 152.90655517578125720 ]721 ],722 [723 [724 98.51061248779297,725 297.78729248046875,726 153.5872344970703727 ],728 [729 98.4955825805664,730 297.8176574707031,731 153.69004821777344732 ]733 ],734 [735 [736 98.4955825805664,737 297.8176574707031,738 153.69004821777344739 ],740 [741 98.47238159179688,742 297.9154968261719,743 153.75167846679688744 ]745 ],746 [747 [748 98.4955825805664,749 297.8176574707031,750 153.69004821777344751 ],752 [753 98.4234619140625,754 297.8824462890625,755 153.75787353515625756 ]757 ],758 [759 [760 98.45415496826172,761 297.7464904785156,762 153.54017639160156763 ],764 [765 98.37261199951172,766 297.67401123046875,767 153.5352020263672768 ]769 ],770 [771 [772 98.37261199951172,773 297.67401123046875,774 153.5352020263672775 ],776 [777 98.3141860961914,778 297.61407470703125,779 153.27279663085938780 ]781 ],782 [783 [784 98.3141860961914,785 297.61407470703125,786 153.27279663085938787 ],788 [789 98.25213623046875,790 297.6591491699219,791 153.01516723632812792 ]793 ],794 [795 [796 98.25213623046875,797 297.6591491699219,798 153.01516723632812799 ],800 [801 98.24822235107422,802 297.68865966796875,803 152.98118591308594804 ]805 ],806 [807 [808 98.25213623046875,809 297.6591491699219,810 153.01516723632812811 ],812 [813 98.22602081298828,814 297.6886291503906,815 152.93289184570312816 ]817 ],818 [819 [820 98.25213623046875,821 297.6591491699219,822 153.01516723632812823 ],824 [825 98.22172546386719,826 297.6620788574219,827 152.9342803955078828 ]829 ],830 [831 [832 98.25213623046875,833 297.6591491699219,834 153.01516723632812835 ],836 [837 98.2310562133789,838 297.6431579589844,839 152.934814453125840 ]841 ],842 [843 [844 98.25213623046875,845 297.6591491699219,846 153.01516723632812847 ],848 [849 98.2455825805664,850 297.62652587890625,851 152.94175720214844852 ]853 ],854 [855 [856 98.24822235107422,857 297.68865966796875,858 152.98118591308594859 ],860 [861 98.2596206665039,862 297.7238464355469,863 152.96405029296875864 ]865 ],866 [867 [868 98.2596206665039,869 297.7238464355469,870 152.96405029296875871 ],872 [873 98.26471710205078,874 297.72698974609375,875 152.93484497070312876 ]877 ],878 [879 [880 98.26471710205078,881 297.72698974609375,882 152.93484497070312883 ],884 [885 98.26542663574219,886 297.72943115234375,887 152.9070587158203888 ]889 ],890 [891 [892 98.22602081298828,893 297.6886291503906,894 152.93289184570312895 ],896 [897 98.22600555419922,898 297.6926574707031,899 152.88658142089844900 ]901 ],902 [903 [904 98.22600555419922,905 297.6926574707031,906 152.88658142089844907 ],908 [909 98.23794555664062,910 297.6935729980469,911 152.85670471191406912 ]913 ],914 [915 [916 98.23794555664062,917 297.6935729980469,918 152.85670471191406919 ],920 [921 98.25003814697266,922 297.6932678222656,923 152.8390350341797924 ]925 ],926 [927 [928 98.22172546386719,929 297.6620788574219,930 152.9342803955078931 ],932 [933 98.23141479492188,934 297.6636657714844,935 152.88800048828125936 ]937 ],938 [939 [940 98.23141479492188,941 297.6636657714844,942 152.88800048828125943 ],944 [945 98.24774169921875,946 297.66650390625,947 152.8636016845703948 ]949 ],950 [951 [952 98.24774169921875,953 297.66650390625,954 152.8636016845703955 ],956 [957 98.26789855957031,958 297.66912841796875,959 152.84332275390625960 ]961 ],962 [963 [964 98.2310562133789,965 297.6431579589844,966 152.934814453125967 ],968 [969 98.24484252929688,970 297.6401062011719,971 152.89712524414062972 ]973 ],974 [975 [976 98.24484252929688,977 297.6401062011719,978 152.89712524414062979 ],980 [981 98.26583862304688,982 297.6419372558594,983 152.8755340576172984 ]985 ],986 [987 [988 98.26583862304688,989 297.6419372558594,990 152.8755340576172991 ],992 [993 98.286865234375,994 297.64447021484375,995 152.86122131347656996 ]997 ],998 [999 [1000 98.2455825805664,1001 297.62652587890625,1002 152.941757202148441003 ],1004 [1005 98.25952911376953,1006 297.6249084472656,1007 152.916763305664061008 ]1009 ],1010 [1011 [1012 98.25952911376953,1013 297.6249084472656,1014 152.916763305664061015 ],1016 [1017 98.27628326416016,1018 297.6255798339844,1019 152.901260375976561020 ]1021 ],1022 [1023 [1024 98.27628326416016,1025 297.6255798339844,1026 152.901260375976561027 ],1028 [1029 98.28997039794922,1030 297.6274719238281,1031 152.88981628417971032 ]1033 ],1034 [1035 [1036 98.4283447265625,1037 297.76739501953125,1038 153.0354003906251039 ],1040 [1041 98.39866638183594,1042 297.8064880371094,1043 152.575256347656251044 ]1045 ],1046 [1047 [1048 98.39866638183594,1049 297.8064880371094,1050 152.575256347656251051 ],1052 [1053 98.4386978149414,1054 297.7596435546875,1055 152.144500732421881056 ]1057 ],1058 [1059 [1060 98.4386978149414,1061 297.7596435546875,1062 152.144500732421881063 ],1064 [1065 98.34420776367188,1066 297.81500244140625,1067 152.110946655273441068 ]1069 ],1070 [1071 [1072 98.34420776367188,1073 297.81500244140625,1074 152.110946655273441075 ],1076 [1077 98.30487823486328,1078 297.8370361328125,1079 152.10807800292971080 ]1081 ],1082 [1083 [1084 98.55254364013672,1085 297.8653564453125,1086 153.035156251087 ],1088 [1089 98.5112075805664,1090 297.88934326171875,1091 152.57487487792971092 ]1093 ],1094 [1095 [1096 98.5112075805664,1097 297.88934326171875,1098 152.57487487792971099 ],1100 [1101 98.54106903076172,1102 297.845458984375,1103 152.14299011230471104 ]1105 ],1106 [1107 [1108 98.54106903076172,1109 297.845458984375,1110 152.14299011230471111 ],1112 [1113 98.50144958496094,1114 297.9480895996094,1115 152.1110839843751116 ]1117 ],1118 [1119 [1120 98.50144958496094,1121 297.9480895996094,1122 152.1110839843751123 ],1124 [1125 98.48602294921875,1126 297.990478515625,1127 152.108856201171881128 ]1129 ]1130 ],1131 "road_id": 221,1132 "lane_id": -2,1133 "section_id": 0,1134 "world2ped": [1135 [1136 -0.6486782431602478,1137 0.7609626650810242,1138 0.012344760820269585,1139 -164.662445068359381140 ],1141 [1142 -0.7610206604003906,1143 -0.6487276554107666,1144 4.6566123179658803e-10,1145 268.14578247070311146 ],1147 [1148 0.008008387871086597,1149 -0.00939461775124073,1150 0.999923825263977,1151 -150.976654052734381152 ],1153 [1154 0.0,1155 0.0,1156 0.0,1157 1.01158 ]1159 ],1160 "position": [1161 4.621706612622461,1162 -1.7784961598783866,1163 0.91799066649994641164 ],1165 "yaw": 4.7132244140186471166 },1167 "position": [1168 4.621706612622461,1169 -1.7784961598783866,1170 0.91799066649994641171 ],1172 "distance": 4.934370660645002,1173 "speed": [1174 4.621706612622461,1175 -1.7784961598783866,1176 0.91799066649994641177 ],1178 "same_dir": true,1179 "is_sign": false1180 }1181 },1182 "QA": {1183 "perception": [1184 {1185 "Q": "What are the important objects in the scene?",1186 "A": "The important objects in the scene are the navy car and the pedestrian to the front of the ego vehicle.",1187 "C": null,1188 "qid": 18,1189 "con_up": -1,1190 "con_down": -1,1191 "cluster": 0,1192 "layer": 0,1193 "object_tags": [1194 "<c7573<CAM_FRONT,805.0,491.7>>",1195 "<c7537<CAM_FRONT,263.6,664.8>>"1196 ]1197 },1198 {1199 "Q": "What are the important objects in the scene? List them from most to least important.",1200 "A": "The pedestrian to the front of the ego vehicle(<c7537<CAM_FRONT,263.6,664.8>>), the navy car that is to the front of the ego vehicle(<c7573<CAM_FRONT,805.0,491.7>>).",