Telkwevr/Bench2Drive-VL-base
Bench2Drive-VL: Full-Stack Software for Closed-Loop Autonomous Driving with Vision Language Models Project Page | GitHub | Paper Bench2Drive-VL is a comprehensive closed-loop benchmark for Vision-Language Models in Autonomous Driving (VLM4AD). It extends the Bench2Drive benchmark by introducing closed-loop evaluation and the DriveCommenter expert model for automated annotation. This repository contains the natural language annotations for the Bench2Drive-Base1000 dataset. These… See the full description on the dataset page: https://huggingface.co/datasets/Telkwevr/Bench2Drive-VL-base.
19.9k
1{2 "key_object_infos": {},3 "QA": {4 "perception": [5 {6 "Q": "What are the important objects in the scene?",7 "A": "There is no important object in the scene.",8 "C": null,9 "qid": 18,10 "con_up": -1,11 "con_down": -1,12 "cluster": 0,13 "layer": 0,14 "object_tags": []15 },16 {17 "Q": "What are the important objects in the scene? List them from most to least important.",18 "A": "There is no important object in the scene.",19 "C": null,20 "qid": 19,21 "con_up": -1,22 "con_down": -1,23 "cluster": 0,24 "layer": 0,25 "object_tags": []26 },27 {28 "Q": "What are the important vehicles in the scene and where on the road are they located? List them from most to least important.",29 "A": "There's no important vehicle in the current scene.",30 "C": null,31 "qid": 26,32 "con_up": [33 [34 4,35 336 ]37 ],38 "con_down": [39 [40 4,41 042 ]43 ],44 "cluster": 4,45 "layer": 1,46 "object_tags": []47 },48 {49 "Q": "There's no important vehicle in the current scene. Where on the road are they located?",50 "A": "There's no important vehicle in the current scene.",51 "C": null,52 "qid": 27,53 "con_up": [54 [55 4,56 357 ]58 ],59 "con_down": [60 [61 4,62 063 ]64 ],65 "cluster": 4,66 "layer": 1,67 "object_tags": []68 },69 {70 "Q": "Is the ego vehicle at a junction?",71 "A": "No, the ego vehicle is not at a junction.",72 "C": null,73 "qid": 35,74 "con_up": [75 [76 1,77 078 ],79 [80 2,81 082 ],83 [84 3,85 186 ],87 [88 3,89 290 ],91 [92 3,93 394 ],95 [96 4,97 098 ]99 ],100 "con_down": -1,101 "cluster": 3,102 "layer": 0,103 "object_tags": []104 },105 {106 "Q": "How many lanes are there in the same direction as the ego car?",107 "A": "There is one lane in the same direction.",108 "C": null,109 "qid": 34,110 "con_up": [111 [112 3,113 1114 ],115 [116 3,117 3118 ],119 [120 3,121 4122 ],123 [124 3,125 5126 ],127 [128 3,129 6130 ],131 [132 3,133 7134 ],135 [136 4,137 0138 ]139 ],140 "con_down": [141 [142 3,143 0144 ]145 ],146 "cluster": 3,147 "layer": 2,148 "object_tags": []149 },150 {151 "Q": "How many lanes are there in the opposite direction to the ego car?",152 "A": "There are no lanes in the opposite direction.",153 "C": null,154 "qid": 34,155 "con_up": [156 [157 3,158 1159 ],160 [161 3,162 3163 ],164 [165 3,166 4167 ],168 [169 3,170 5171 ],172 [173 3,174 6175 ],176 [177 3,178 7179 ],180 [181 4,182 0183 ]184 ],185 "con_down": [186 [187 3,188 0189 ]190 ],191 "cluster": 3,192 "layer": 2,193 "object_tags": []194 },195 {196 "Q": "On which lane is the ego vehicle (left most lane of the lanes going in the same direction is indicated with 0)?",197 "A": "The ego vehicle is on lane 0.",198 "C": null,199 "qid": 33,200 "con_up": [201 [202 3,203 1204 ],205 [206 3,207 4208 ],209 [210 3,211 5212 ],213 [214 3,215 6216 ],217 [218 4,219 0220 ]221 ],222 "con_down": [223 [224 3,225 0226 ]227 ],228 "cluster": 3,229 "layer": 3,230 "object_tags": []231 },232 {233 "Q": "What lane marking is on the left side of the ego car?",234 "A": "There is no lane marking on the left side of the ego car.",235 "C": null,236 "qid": 32,237 "con_up": [238 [239 3,240 5241 ],242 [243 3,244 6245 ]246 ],247 "con_down": [248 [249 3,250 2251 ],252 [253 3,254 3255 ]256 ],257 "cluster": 3,258 "layer": 4,259 "object_tags": []260 },261 {262 "Q": "What lane marking is on the right side of the ego car?",263 "A": "There is no lane marking on the right side of the ego car.",264 "C": null,265 "qid": 32,266 "con_up": [267 [268 3,269 5270 ],271 [272 3,273 6274 ]275 ],276 "con_down": [277 [278 3,279 2280 ],281 [282 3,283 3284 ]285 ],286 "cluster": 3,287 "layer": 4,288 "object_tags": []289 },290 {291 "Q": "Describe the current lane's direction.",292 "A": "The current lane is going straight.",293 "C": null,294 "qid": 44,295 "con_up": -1,296 "con_down": -1,297 "cluster": 3,298 "layer": 6,299 "object_tags": []300 },301 {302 "Q": "Is the ego vehicle affected by a traffic light?",303 "A": "No, the ego vehicle is not affected by a traffic light.",304 "C": null,305 "qid": 5,306 "con_up": [307 [308 2,309 1310 ]311 ],312 "con_down": [313 [314 3,315 0316 ]317 ],318 "cluster": 2,319 "layer": 0,320 "object_tags": []321 },322 {323 "Q": "How many pedestrians are there?",324 "A": "There are no pedestrians.",325 "C": null,326 "qid": 1,327 "con_up": [328 [329 6,330 0331 ]332 ],333 "con_down": -1,334 "cluster": 5,335 "layer": 0,336 "object_tags": []337 },338 {339 "Q": "What is current time and weather?",340 "A": "It is noon, raining lightly.",341 "C": null,342 "qid": 37,343 "con_up": -1,344 "con_down": -1,345 "cluster": 3,346 "layer": 1,347 "object_tags": []348 },349 {350 "Q": "Is there an obstacle on the current road?",351 "A": "No, there is no obstacle on the current route.",352 "C": null,353 "qid": 11,354 "con_up": [355 [356 3,357 8358 ]359 ],360 "con_down": -1,361 "cluster": 3,362 "layer": 9,363 "object_tags": []364 },365 {366 "Q": "What is the current speed limit?",367 "A": "There's no speed limit now.",368 "C": null,369 "qid": 7,370 "con_up": [371 [372 6,373 0374 ]375 ],376 "con_down": [377 [378 3,379 2380 ]381 ],382 "cluster": 3,383 "layer": 7,384 "object_tags": []385 }386 ],387 "prediction": [388 {389 "Q": "There's no important vehicle in the current scene. What are the rough moving speed and moving direction of them?",390 "A": "There's no important vehicle in the current scene.",391 "C": null,392 "qid": 24,393 "con_up": [394 [395 4,396 3397 ]398 ],399 "con_down": [400 [401 4,402 0403 ]404 ],405 "cluster": 4,406 "layer": 1,407 "object_tags": []408 },409 {410 "Q": "There's no important vehicle in the current scene. What are the exact moving speed and moving direction of them?",411 "A": "There's no important vehicle in the current scene.",412 "C": null,413 "qid": 25,414 "con_up": [415 [416 4,417 3418 ]419 ],420 "con_down": [421 [422 4,423 0424 ]425 ],426 "cluster": 4,427 "layer": 1,428 "object_tags": []429 },430 {431 "Q": "The ego vehicle wants to follow the road. Apart from its lane, which lanes are important to watch out for?",432 "A": "There are no other driving lanes to watch out for since the ego vehicle is on a one lane road.",433 "C": null,434 "qid": 36,435 "con_up": [436 [437 4,438 3439 ]440 ],441 "con_down": [442 [443 3,444 0445 ],446 [447 3,448 2449 ],450 [451 3,452 3453 ]454 ],455 "cluster": 3,456 "layer": 1,457 "object_tags": []458 },459 {460 "Q": "In which direction is the ego car allowed to change lanes?",461 "A": "The ego vehicle can not change lanes since it is on a one lane road.",462 "C": null,463 "qid": 31,464 "con_up": [465 [466 6,467 0468 ]469 ],470 "con_down": [471 [472 3,473 2474 ],475 [476 3,477 3478 ],479 [480 3,481 4482 ]483 ],484 "cluster": 3,485 "layer": 5,486 "object_tags": []487 },488 {489 "Q": "From which side are other vehicles allowed to change lanes into the ego lane?",490 "A": "There are no lane changes possible since the ego vehicle is on a one lane road.",491 "C": null,492 "qid": 30,493 "con_up": [494 [495 6,496 0497 ]498 ],499 "con_down": [500 [501 3,502 2503 ],504 [505 3,506 3507 ],508 [509 3,510 4511 ],512 [513 3,514 5515 ]516 ],517 "cluster": 3,518 "layer": 6,519 "object_tags": []520 },521 {522 "Q": "Is the ego vehicle affected by a stop sign?",523 "A": "No, the ego vehicle is not affected by a stop sign.",524 "C": null,525 "qid": 2,526 "con_up": [527 [528 1,529 1530 ]531 ],532 "con_down": [533 [534 3,535 0536 ]537 ],538 "cluster": 1,539 "layer": 0,540 "object_tags": []541 },542 {543 "Q": "Is the ego vehicle affected by a speed limit sign?",544 "A": "No, the ego vehicle is not affected by a speed limit sign.",545 "C": null,546 "qid": 3,547 "con_up": [548 [549 1,550 1551 ]552 ],553 "con_down": [554 [555 3,556 0557 ]558 ],559 "cluster": 1,560 "layer": 0,561 "object_tags": []562 },563 {564 "Q": "List the traffic signs affecting the ego vehicle in the current scenario.",565 "A": "There's no traffic sign affecting the ego vehicle.",566 "C": null,567 "qid": 4,568 "con_up": [569 [570 1,571 1572 ]573 ],574 "con_down": [575 [576 3,577 0578 ]579 ],580 "cluster": 1,581 "layer": 0,582 "object_tags": []583 },584 {585 "Q": "What is the state of the traffic light?",586 "A": "There is no traffic light affecting the ego vehicle.",587 "C": null,588 "qid": 6,589 "con_up": [590 [591 2,592 2593 ]594 ],595 "con_down": [596 [597 2,598 0599 ]600 ],601 "cluster": 2,602 "layer": 1,603 "object_tags": []604 },605 {606 "Q": "What is current time and weather? What hazards might it bring?",607 "A": "It is noon, raining lightly. There's no hazard caused by time or weather. The puddles on the road may cause vehicles to skid or even lose control.",608 "C": null,609 "qid": 38,610 "con_up": -1,611 "con_down": -1,612 "cluster": 3,613 "layer": 1,614 "object_tags": []615 },616 {617 "Q": "Apart from vehicles on the road, visible pedestrians and the weather, what other factors in the current scenario could pose potential hazards?",618 "A": "The puddles on the road will make vehicle more difficult to control.",619 "C": null,620 "qid": 40,621 "con_up": -1,622 "con_down": -1,623 "cluster": 3,624 "layer": 1,625 "object_tags": []626 }627 ],628 "planning": [629 {630 "Q": "The ego vehicle follows the road. There's no important vehicle in the current scene. Among them, please identify those that may overlap with the ego vehicle's path and provide reasons for the overlap.",631 "A": "There's no important vehicle, so none of them potentially crossing the path of ego vehicle now.",632 "C": null,633 "qid": 28,634 "con_up": [635 [636 4,637 3638 ]639 ],640 "con_down": [641 [642 4,643 0644 ]645 ],646 "cluster": 4,647 "layer": 1,648 "object_tags": []649 },650 {651 "Q": "The ego vehicle follows the road. There's no important vehicle in the current scene. Among them, please identify those that may overlap with the ego vehicle's path.",652 "A": "There's no important vehicle, so none of them potentially crossing the path of ego vehicle now.",653 "C": null,654 "qid": 29,655 "con_up": [656 [657 4,658 3659 ]660 ],661 "con_down": [662 [663 4,664 0665 ]666 ],667 "cluster": 4,668 "layer": 1,669 "object_tags": []670 },671 {672 "Q": "The ego vehicle follows the road. There's no important vehicle in the current scene. Among them, identify those that may overlap with the ego vehicle's path and the actions that could lead to a collision.",673 "A": "There's no important vehicle, so none of them potentially crossing the path of ego vehicle now.",674 "C": null,675 "qid": 46,676 "con_up": [677 [678 4,679 3680 ]681 ],682 "con_down": [683 [684 4,685 0686 ]687 ],688 "cluster": 4,689 "layer": 1,690 "object_tags": []691 },692 {693 "Q": "The ego vehicle follows the road. There's no important vehicle in the current scene. Among them, identify those that may overlap with the ego vehicle's path, provide reasons for the overlap and the actions that could lead to a collision.",694 "A": "There's no important vehicle, so none of them potentially crossing the path of ego vehicle now.",695 "C": null,696 "qid": 47,697 "con_up": [698 [699 4,700 3701 ]702 ],703 "con_down": [704 [705 4,706 0707 ]708 ],709 "cluster": 4,710 "layer": 1,711 "object_tags": []712 },713 {714 "Q": "What is current time and weather? What should the ego vehicle do according to them?",715 "A": "It is noon, raining lightly. There's no hazard caused by time or weather. The puddles on the road may cause vehicles to skid or even lose control, so drive carefully, avoid sudden movements.",716 "C": null,717 "qid": 39,718 "con_up": -1,719 "con_down": -1,720 "cluster": 3,721 "layer": 1,722 "object_tags": []723 },724 {725 "Q": "Apart from vehicles on the road, visible pedestrians and the weather, what other factors in the current scenario could pose potential hazards? What strategies should the ego vehicle adopt to address them?",726 "A": "The puddles on the road will make vehicle more difficult to control, so the ego vehicle should drive cautiously and be prepared for potential control loss.",727 "C": null,728 "qid": 41,729 "con_up": -1,730 "con_down": -1,731 "cluster": 3,732 "layer": 1,733 "object_tags": []734 },735 {736 "Q": "Does the ego vehicle need to change lanes or deviate from the lane center due to an upcoming obstruction?",737 "A": "No, the ego vehicle can stay on its current lane.",738 "C": null,739 "qid": 10,740 "con_up": [741 [742 6,743 0744 ]745 ],746 "con_down": [747 [748 3,749 9750 ]751 ],752 "cluster": 3,753 "layer": 8,754 "object_tags": []755 },756 {757 "Q": "Does the ego vehicle need to change lanes or deviate from the lane for reasons other than the upcoming obstruction? Why?",758 "A": "No, no other reason supports the ego vehicle to change lane.",759 "C": null,760 "qid": 12,761 "con_up": [762 [763 6,764 0765 ]766 ],767 "con_down": [768 [769 3,770 9771 ]772 ],773 "cluster": 3,774 "layer": 8,775 "object_tags": []776 },777 {778 "Q": "Must the ego vehicle change lane or deviate from the lane now? why?",779 "A": "No, the ego vehicle can stay on its current lane.",780 "C": null,781 "qid": 13,782 "con_up": [783 [784 6,785 0786 ]787 ],788 "con_down": [789 [790 1,791 0792 ]793 ],794 "cluster": 1,795 "layer": 1,796 "object_tags": []797 },798 {799 "Q": "What should the ego vehicle do based on the traffic light?",800 "A": "There is no traffic light affecting the ego vehicle.",801 "C": null,802 "qid": 9,803 "con_up": [804 [805 6,806 0807 ]808 ],809 "con_down": [810 [811 2,812 1813 ]814 ],815 "cluster": 2,816 "layer": 2,817 "object_tags": []818 },819 {820 "Q": "Does the ego vehicle need to brake? Why?",821 "A": "There is no reason for the ego vehicle to brake.",822 "C": null,823 "qid": 8,824 "con_up": -1,825 "con_down": [826 [827 1,828 1829 ],830 [831 2,832 2833 ],834 [835 3,836 5837 ],838 [839 3,840 6841 ],842 [843 3,844 7845 ],846 [847 3,848 8849 ],850 [851 4,852 3853 ],854 [855 5,856 0857 ]858 ],859 "cluster": 6,860 "layer": 0,861 "object_tags": []862 },863 {864 "Q": "Identify all traffic lights and signs affecting the ego vehicle in current scene. Based on these traffic signs, what actions should the ego vehicle take respectively?",865 "A": "There's no traffic light or sign affecting the ego vehicle right now.",866 "C": null,867 "qid": 15,868 "con_up": [869 [870 6,871 0872 ]873 ],874 "con_down": [875 [876 1,877 0878 ]879 ],880 "cluster": 1,881 "layer": 1,882 "object_tags": []883 }884 ],885 "behaviour": [886 {887 "Q": "What is the correct action for the ego vehicle to take now?",888 "A": "The ego vehicle should continue driving along the current lane.",889 "C": null,890 "qid": 43,891 "con_up": -1,892 "con_down": -1,893 "cluster": 3,894 "layer": 1,895 "object_tags": []896 },897 {898 "Q": "Provide your planned trajectory for the ego vehicle using waypoints in the ego vehicle's coordinate system. Generate one waypoint every 0.5 seconds. Example output:\n{\n \"0.5s\": [x1, y1],\n \"1.0s\": [x2, y2],\n \"1.5s\": [x3, y3],\n \"2.0s\": [x4, y4],\n \"2.5s\": [x5, y5],\n \"3.0s\": [x6, y6],\n \"3.5s\": [x7, y7],\n \"4.0s\": [x8, y8],\n}",899 "A": "{\"0.5s\": [4.01, -0.0], \"1.0s\": [8.1, 0.0], \"1.5s\": [11.99, 0.0], \"2.0s\": [15.91, 0.0], \"2.5s\": [19.59, 0.0], \"3.0s\": [23.79, 0.0], \"3.5s\": [27.33, 0.0], \"4.0s\": [30.82, 0.0]}",900 "C": null,901 "qid": 42,902 "con_up": -1,903 "con_down": -1,904 "cluster": 3,905 "layer": 1,906 "object_tags": []907 },908 {909 "Q": "Provide the appropriate behavior for the ego vehicle, which consists of two keys: the Direction key, which can be 'FOLLOW_LANE', 'CHANGE_LANE_LEFT', 'CHANGE_LANE_RIGHT', 'GO_STRAIGHT', 'TURN_LEFT', or 'TURN_RIGHT'; and the Speed key, which can be 'KEEP', 'ACCELERATE', 'DECELERATE', or 'STOP'.",910 "A": "FOLLOW_LANE, KEEP",911 "C": null,912 "qid": 50,913 "con_up": -1,914 "con_down": -1,915 "cluster": 3,916 "layer": 1,917 "object_tags": []918 }919 ]920 },921 "extra_flags": {922 "ego_speed": 7.181398489196382,923 "change_lane_flag": false,924 "change_lane_gt": false,925 "speed_cmd": "KEEP",926 "direction_cmd": "FOLLOW_LANE",927 "speed_change_gt": "Accelerate",928 "waiting_for_red_light": false,929 "is_trivial_case": true930 },931 "image_paths": {932 "CAM_FRONT": "./.worker_3_cache/DynamicObjectCrossing_Town15_Route28_Weather2/DynamicObjectCrossing_Town15_Route28_Weather2/camera/rgb_front/00045.jpg",933 "CAM_FRONT_LEFT": "./.worker_3_cache/DynamicObjectCrossing_Town15_Route28_Weather2/DynamicObjectCrossing_Town15_Route28_Weather2/camera/rgb_front_left/00045.jpg",934 "CAM_FRONT_RIGHT": "./.worker_3_cache/DynamicObjectCrossing_Town15_Route28_Weather2/DynamicObjectCrossing_Town15_Route28_Weather2/camera/rgb_front_right/00045.jpg",935 "CAM_BACK": "./.worker_3_cache/DynamicObjectCrossing_Town15_Route28_Weather2/DynamicObjectCrossing_Town15_Route28_Weather2/camera/rgb_back/00045.jpg",936 "CAM_BACK_LEFT": "./.worker_3_cache/DynamicObjectCrossing_Town15_Route28_Weather2/DynamicObjectCrossing_Town15_Route28_Weather2/camera/rgb_back_left/00045.jpg",937 "CAM_BACK_RIGHT": "./.worker_3_cache/DynamicObjectCrossing_Town15_Route28_Weather2/DynamicObjectCrossing_Town15_Route28_Weather2/camera/rgb_back_right/00045.jpg"938 }939}