OfficerChul/Android-Control-84k
Android Control Dataset Overview This directory contains two dataset files (and_ctrl_train.json and and_ctrl_test.json) derived from the Android Control project by Google Research. These datasets have been formatted specifically for GUI grounding training in LLaMA-Factory. Dataset Description The Android Control dataset consists of episodes where each episode contains multiple steps. Each step includes: Step instructions: Natural language… See the full description on the dataset page: https://huggingface.co/datasets/OfficerChul/Android-Control-84k.
259
1[2 {3 "messages": [4 {5 "role": "system",6 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."7 },8 {9 "role": "user",10 "content": "<image>Click on the Recording 2 "11 },12 {13 "role": "assistant",14 "content": "{\"action_type\": \"click\", \"x\": 561, \"y\": 535}"15 }16 ],17 "images": [18 "and_ctrl/out_episode_18557_step_001.png"19 ]20 },21 {22 "messages": [23 {24 "role": "system",25 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."26 },27 {28 "role": "user",29 "content": "<image>Scroll down to view the price of other sellers of this shirt."30 },31 {32 "role": "assistant",33 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"34 }35 ],36 "images": [37 "and_ctrl/out_episode_13740_step_001.png"38 ]39 },40 {41 "messages": [42 {43 "role": "system",44 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."45 },46 {47 "role": "user",48 "content": "<image>Swipe up to view the category section"49 },50 {51 "role": "assistant",52 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"53 }54 ],55 "images": [56 "and_ctrl/out_episode_17348_step_005.png"57 ]58 },59 {60 "messages": [61 {62 "role": "system",63 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."64 },65 {66 "role": "user",67 "content": "<image>Enter Pride and Prejudice in the text box."68 },69 {70 "role": "assistant",71 "content": "{\"action_type\": \"input_text\", \"text\": \"Pride and prejudice\"}"72 }73 ],74 "images": [75 "and_ctrl/out_episode_13626_step_003.png"76 ]77 },78 {79 "messages": [80 {81 "role": "system",82 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."83 },84 {85 "role": "user",86 "content": "<image>Long press on the mail received from Yummly."87 },88 {89 "role": "assistant",90 "content": "{\"action_type\": \"long_press\", \"x\": 571, \"y\": 723}"91 }92 ],93 "images": [94 "and_ctrl/out_episode_15290_step_000.png"95 ]96 },97 {98 "messages": [99 {100 "role": "system",101 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."102 },103 {104 "role": "user",105 "content": "<image>Long press on the How are you, Amelia? to view the forward to inbox option."106 },107 {108 "role": "assistant",109 "content": "{\"action_type\": \"long_press\", \"x\": 779, \"y\": 1153}"110 }111 ],112 "images": [113 "and_ctrl/out_episode_2500_step_000.png"114 ]115 },116 {117 "messages": [118 {119 "role": "system",120 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."121 },122 {123 "role": "user",124 "content": "<image>Go back to the previous page"125 },126 {127 "role": "assistant",128 "content": "{\"action_type\": \"navigate_back\"}"129 }130 ],131 "images": [132 "and_ctrl/out_episode_13409_step_001.png"133 ]134 },135 {136 "messages": [137 {138 "role": "system",139 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."140 },141 {142 "role": "user",143 "content": "<image>Long press on the message I'm good, what about you, Natalia? to view the forward option of this message."144 },145 {146 "role": "assistant",147 "content": "{\"action_type\": \"long_press\", \"x\": 554, \"y\": 637}"148 }149 ],150 "images": [151 "and_ctrl/out_episode_12285_step_000.png"152 ]153 },154 {155 "messages": [156 {157 "role": "system",158 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."159 },160 {161 "role": "user",162 "content": "<image>Open Easy Currency Converter app."163 },164 {165 "role": "assistant",166 "content": "{\"action_type\": \"open_app\", \"app_name\": \"Currency\"}"167 }168 ],169 "images": [170 "and_ctrl/out_episode_15931_step_000.png"171 ]172 },173 {174 "messages": [175 {176 "role": "system",177 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."178 },179 {180 "role": "user",181 "content": "<image>Type Bananas in search bar"182 },183 {184 "role": "assistant",185 "content": "{\"action_type\": \"input_text\", \"text\": \"Bananas\"}"186 }187 ],188 "images": [189 "and_ctrl/out_episode_11805_step_003.png"190 ]191 },192 {193 "messages": [194 {195 "role": "system",196 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."197 },198 {199 "role": "user",200 "content": "<image>Search for spanner set"201 },202 {203 "role": "assistant",204 "content": "{\"action_type\": \"input_text\", \"text\": \"Spanner set\"}"205 }206 ],207 "images": [208 "and_ctrl/out_episode_3067_step_001.png"209 ]210 },211 {212 "messages": [213 {214 "role": "system",215 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."216 },217 {218 "role": "user",219 "content": "<image>Click on the Done button on the bottom of the screen."220 },221 {222 "role": "assistant",223 "content": "{\"action_type\": \"click\", \"x\": 540, \"y\": 2232}"224 }225 ],226 "images": [227 "and_ctrl/out_episode_12215_step_013.png"228 ]229 },230 {231 "messages": [232 {233 "role": "system",234 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."235 },236 {237 "role": "user",238 "content": "<image>Click on the USD tab"239 },240 {241 "role": "assistant",242 "content": "{\"action_type\": \"wait\"}"243 }244 ],245 "images": [246 "and_ctrl/out_episode_2762_step_002.png"247 ]248 },249 {250 "messages": [251 {252 "role": "system",253 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."254 },255 {256 "role": "user",257 "content": "<image>Long press on the Google community mail"258 },259 {260 "role": "assistant",261 "content": "{\"action_type\": \"long_press\", \"x\": 761, \"y\": 2347}"262 }263 ],264 "images": [265 "and_ctrl/out_episode_1487_step_000.png"266 ]267 },268 {269 "messages": [270 {271 "role": "system",272 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."273 },274 {275 "role": "user",276 "content": "<image>open the Notes pad app"277 },278 {279 "role": "assistant",280 "content": "{\"action_type\": \"open_app\", \"app_name\": \"Notepad\"}"281 }282 ],283 "images": [284 "and_ctrl/out_episode_12531_step_000.png"285 ]286 },287 {288 "messages": [289 {290 "role": "system",291 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."292 },293 {294 "role": "user",295 "content": "<image>Open the wordweb app."296 },297 {298 "role": "assistant",299 "content": "{\"action_type\": \"open_app\", \"app_name\": \"WordWeb\"}"300 }301 ],302 "images": [303 "and_ctrl/out_episode_8798_step_000.png"304 ]305 },306 {307 "messages": [308 {309 "role": "system",310 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."311 },312 {313 "role": "user",314 "content": "<image>Long press on the second recording"315 },316 {317 "role": "assistant",318 "content": "{\"action_type\": \"long_press\", \"x\": 561, \"y\": 535}"319 }320 ],321 "images": [322 "and_ctrl/out_episode_10025_step_000.png"323 ]324 },325 {326 "messages": [327 {328 "role": "system",329 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."330 },331 {332 "role": "user",333 "content": "<image>Click on the Amera from the suggestions."334 },335 {336 "role": "assistant",337 "content": "{\"action_type\": \"wait\"}"338 }339 ],340 "images": [341 "and_ctrl/out_episode_19969_step_002.png"342 ]343 },344 {345 "messages": [346 {347 "role": "system",348 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."349 },350 {351 "role": "user",352 "content": "<image>Long press on the backspace key of the key board to clear the search bar."353 },354 {355 "role": "assistant",356 "content": "{\"action_type\": \"long_press\", \"x\": 994, \"y\": 2001}"357 }358 ],359 "images": [360 "and_ctrl/out_episode_13389_step_004.png"361 ]362 },363 {364 "messages": [365 {366 "role": "system",367 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."368 },369 {370 "role": "user",371 "content": "<image>Search for apple iPhone 13"372 },373 {374 "role": "assistant",375 "content": "{\"action_type\": \"input_text\", \"text\": \"apple iPhone 13\"}"376 }377 ],378 "images": [379 "and_ctrl/out_episode_4379_step_003.png"380 ]381 },382 {383 "messages": [384 {385 "role": "system",386 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."387 },388 {389 "role": "user",390 "content": "<image>Go to the home screen"391 },392 {393 "role": "assistant",394 "content": "{\"action_type\": \"navigate_home\"}"395 }396 ],397 "images": [398 "and_ctrl/out_episode_2284_step_000.png"399 ]400 },401 {402 "messages": [403 {404 "role": "system",405 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."406 },407 {408 "role": "user",409 "content": "<image>Swipe up to view the product information"410 },411 {412 "role": "assistant",413 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"414 }415 ],416 "images": [417 "and_ctrl/out_episode_18054_step_011.png"418 ]419 },420 {421 "messages": [422 {423 "role": "system",424 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."425 },426 {427 "role": "user",428 "content": "<image>Open the trivago app."429 },430 {431 "role": "assistant",432 "content": "{\"action_type\": \"open_app\", \"app_name\": \"trivago\"}"433 }434 ],435 "images": [436 "and_ctrl/out_episode_2458_step_000.png"437 ]438 },439 {440 "messages": [441 {442 "role": "system",443 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."444 },445 {446 "role": "user",447 "content": "<image>go back to the previous page"448 },449 {450 "role": "assistant",451 "content": "{\"action_type\": \"navigate_back\"}"452 }453 ],454 "images": [455 "and_ctrl/out_episode_20135_step_001.png"456 ]457 },458 {459 "messages": [460 {461 "role": "system",462 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."463 },464 {465 "role": "user",466 "content": "<image>Click on the Continue button at the bottom of the screen to complete other task of this lesson "467 },468 {469 "role": "assistant",470 "content": "{\"action_type\": \"click\", \"x\": 540, \"y\": 2226}"471 }472 ],473 "images": [474 "and_ctrl/out_episode_11422_step_000.png"475 ]476 },477 {478 "messages": [479 {480 "role": "system",481 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."482 },483 {484 "role": "user",485 "content": "<image>Type badminton shoes in the search bar"486 },487 {488 "role": "assistant",489 "content": "{\"action_type\": \"input_text\", \"text\": \"badminton shoes\"}"490 }491 ],492 "images": [493 "and_ctrl/out_episode_13184_step_002.png"494 ]495 },496 {497 "messages": [498 {499 "role": "system",500 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."501 },502 {503 "role": "user",504 "content": "<image>go back to previous page"505 },506 {507 "role": "assistant",508 "content": "{\"action_type\": \"navigate_back\"}"509 }510 ],511 "images": [512 "and_ctrl/out_episode_3207_step_000.png"513 ]514 },515 {516 "messages": [517 {518 "role": "system",519 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."520 },521 {522 "role": "user",523 "content": "<image>Open the Coursera App"524 },525 {526 "role": "assistant",527 "content": "{\"action_type\": \"wait\"}"528 }529 ],530 "images": [531 "and_ctrl/out_episode_17428_step_001.png"532 ]533 },534 {535 "messages": [536 {537 "role": "system",538 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."539 },540 {541 "role": "user",542 "content": "<image>go back to the previous page"543 },544 {545 "role": "assistant",546 "content": "{\"action_type\": \"navigate_back\"}"547 }548 ],549 "images": [550 "and_ctrl/out_episode_8581_step_000.png"551 ]552 },553 {554 "messages": [555 {556 "role": "system",557 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."558 },559 {560 "role": "user",561 "content": "<image>Click on the Nike Elevate 3 shoe"562 },563 {564 "role": "assistant",565 "content": "{\"action_type\": \"wait\"}"566 }567 ],568 "images": [569 "and_ctrl/out_episode_15289_step_001.png"570 ]571 },572 {573 "messages": [574 {575 "role": "system",576 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."577 },578 {579 "role": "user",580 "content": "<image>Long press on the Bubb's contact."581 },582 {583 "role": "assistant",584 "content": "{\"action_type\": \"long_press\", \"x\": 540, \"y\": 1112}"585 }586 ],587 "images": [588 "and_ctrl/out_episode_10600_step_001.png"589 ]590 },591 {592 "messages": [593 {594 "role": "system",595 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."596 },597 {598 "role": "user",599 "content": "<image>click on the jump to recipe"600 },601 {602 "role": "assistant",603 "content": "{\"action_type\": \"wait\"}"604 }605 ],606 "images": [607 "and_ctrl/out_episode_8429_step_012.png"608 ]609 },610 {611 "messages": [612 {613 "role": "system",614 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."615 },616 {617 "role": "user",618 "content": "<image>Go back to the home screen of the app."619 },620 {621 "role": "assistant",622 "content": "{\"action_type\": \"navigate_back\"}"623 }624 ],625 "images": [626 "and_ctrl/out_episode_9907_step_002.png"627 ]628 },629 {630 "messages": [631 {632 "role": "system",633 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."634 },635 {636 "role": "user",637 "content": "<image>Swipe up to view the reviews"638 },639 {640 "role": "assistant",641 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"642 }643 ],644 "images": [645 "and_ctrl/out_episode_4393_step_009.png"646 ]647 },648 {649 "messages": [650 {651 "role": "system",652 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."653 },654 {655 "role": "user",656 "content": "<image>Go back to the previous page to search for the hotel."657 },658 {659 "role": "assistant",660 "content": "{\"action_type\": \"navigate_back\"}"661 }662 ],663 "images": [664 "and_ctrl/out_episode_2141_step_000.png"665 ]666 },667 {668 "messages": [669 {670 "role": "system",671 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."672 },673 {674 "role": "user",675 "content": "<image>Click on the apply button"676 },677 {678 "role": "assistant",679 "content": "{\"action_type\": \"wait\"}"680 }681 ],682 "images": [683 "and_ctrl/out_episode_19758_step_010.png"684 ]685 },686 {687 "messages": [688 {689 "role": "system",690 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."691 },692 {693 "role": "user",694 "content": "<image>Click on the plus icon at the bottom right corner of the screen to view the upload option."695 },696 {697 "role": "assistant",698 "content": "{\"action_type\": \"click\", \"x\": 964, \"y\": 1990}"699 }700 ],701 "images": [702 "and_ctrl/out_episode_13974_step_001.png"703 ]704 },705 {706 "messages": [707 {708 "role": "system",709 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."710 },711 {712 "role": "user",713 "content": "<image>open the Vimeo app"714 },715 {716 "role": "assistant",717 "content": "{\"action_type\": \"open_app\", \"app_name\": \"Vimeo\"}"718 }719 ],720 "images": [721 "and_ctrl/out_episode_5341_step_000.png"722 ]723 },724 {725 "messages": [726 {727 "role": "system",728 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."729 },730 {731 "role": "user",732 "content": "<image>Open the Heart Radio app."733 },734 {735 "role": "assistant",736 "content": "{\"action_type\": \"wait\"}"737 }738 ],739 "images": [740 "and_ctrl/out_episode_7462_step_001.png"741 ]742 },743 {744 "messages": [745 {746 "role": "system",747 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."748 },749 {750 "role": "user",751 "content": "<image>Go back to the previous screen"752 },753 {754 "role": "assistant",755 "content": "{\"action_type\": \"navigate_back\"}"756 }757 ],758 "images": [759 "and_ctrl/out_episode_30_step_000.png"760 ]761 },762 {763 "messages": [764 {765 "role": "system",766 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."767 },768 {769 "role": "user",770 "content": "<image>Go to the Statistics section"771 },772 {773 "role": "assistant",774 "content": "{\"action_type\": \"wait\"}"775 }776 ],777 "images": [778 "and_ctrl/out_episode_18820_step_004.png"779 ]780 },781 {782 "messages": [783 {784 "role": "system",785 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."786 },787 {788 "role": "user",789 "content": "<image>Click on free sample option "790 },791 {792 "role": "assistant",793 "content": "{\"action_type\": \"click\", \"x\": 306, \"y\": 1074}"794 }795 ],796 "images": [797 "and_ctrl/out_episode_6153_step_000.png"798 ]799 },800 {801 "messages": [802 {803 "role": "system",804 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."805 },806 {807 "role": "user",808 "content": "<image>Long press on Rappi email "809 },810 {811 "role": "assistant",812 "content": "{\"action_type\": \"long_press\", \"x\": 571, \"y\": 1714}"813 }814 ],815 "images": [816 "and_ctrl/out_episode_10325_step_000.png"817 ]818 },819 {820 "messages": [821 {822 "role": "system",823 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."824 },825 {826 "role": "user",827 "content": "<image>Open the NoBroker app"828 },829 {830 "role": "assistant",831 "content": "{\"action_type\": \"open_app\", \"app_name\": \"NoBroker\"}"832 }833 ],834 "images": [835 "and_ctrl/out_episode_17079_step_000.png"836 ]837 },838 {839 "messages": [840 {841 "role": "system",842 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."843 },844 {845 "role": "user",846 "content": "<image>Type Creamy Coffee and Cookie in search bar"847 },848 {849 "role": "assistant",850 "content": "{\"action_type\": \"wait\"}"851 }852 ],853 "images": [854 "and_ctrl/out_episode_9394_step_004.png"855 ]856 },857 {858 "messages": [859 {860 "role": "system",861 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."862 },863 {864 "role": "user",865 "content": "<image>Swipe up to view more paintings"866 },867 {868 "role": "assistant",869 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"870 }871 ],872 "images": [873 "and_ctrl/out_episode_4060_step_006.png"874 ]875 },876 {877 "messages": [878 {879 "role": "system",880 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."881 },882 {883 "role": "user",884 "content": "<image>Long press on the back space key of key board to clear the title section."885 },886 {887 "role": "assistant",888 "content": "{\"action_type\": \"long_press\", \"x\": 949, \"y\": 1986}"889 }890 ],891 "images": [892 "and_ctrl/out_episode_20117_step_004.png"893 ]894 },895 {896 "messages": [897 {898 "role": "system",899 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."900 },901 {902 "role": "user",903 "content": "<image>Swipe up to view the 41 size"904 },905 {906 "role": "assistant",907 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"908 }909 ],910 "images": [911 "and_ctrl/out_episode_8256_step_002.png"912 ]913 },914 {915 "messages": [916 {917 "role": "system",918 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."919 },920 {921 "role": "user",922 "content": "<image>open the Virtual Kitchen app"923 },924 {925 "role": "assistant",926 "content": "{\"action_type\": \"wait\"}"927 }928 ],929 "images": [930 "and_ctrl/out_episode_4767_step_001.png"931 ]932 },933 {934 "messages": [935 {936 "role": "system",937 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."938 },939 {940 "role": "user",941 "content": "<image>Enter the name of the product."942 },943 {944 "role": "assistant",945 "content": "{\"action_type\": \"input_text\", \"text\": \"wireless mouse\"}"946 }947 ],948 "images": [949 "and_ctrl/out_episode_18121_step_002.png"950 ]951 },952 {953 "messages": [954 {955 "role": "system",956 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."957 },958 {959 "role": "user",960 "content": "<image>Click on Deadline at the left "961 },962 {963 "role": "assistant",964 "content": "{\"action_type\": \"click\", \"x\": 487, \"y\": 625}"965 }966 ],967 "images": [968 "and_ctrl/out_episode_15040_step_001.png"969 ]970 },971 {972 "messages": [973 {974 "role": "system",975 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."976 },977 {978 "role": "user",979 "content": "<image>click on Settings for other devices option"980 },981 {982 "role": "assistant",983 "content": "{\"action_type\": \"click\", \"x\": 280, \"y\": 1633}"984 }985 ],986 "images": [987 "and_ctrl/out_episode_17001_step_003.png"988 ]989 },990 {991 "messages": [992 {993 "role": "system",994 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."995 },996 {997 "role": "user",998 "content": "<image>click on the Compare button"999 },1000 {1001 "role": "assistant",1002 "content": "{\"action_type\": \"click\", \"x\": 539, \"y\": 1865}"1003 }1004 ],1005 "images": [1006 "and_ctrl/out_episode_9243_step_012.png"1007 ]1008 },1009 {1010 "messages": [1011 {1012 "role": "system",1013 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1014 },1015 {1016 "role": "user",1017 "content": "<image>Swipe up to view the reviews"1018 },1019 {1020 "role": "assistant",1021 "content": "{\"action_type\": \"scroll\", \"direction\": \"down\"}"1022 }1023 ],1024 "images": [1025 "and_ctrl/out_episode_15958_step_004.png"1026 ]1027 },1028 {1029 "messages": [1030 {1031 "role": "system",1032 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1033 },1034 {1035 "role": "user",1036 "content": "<image>Click on the search bar on the top of the screen."1037 },1038 {1039 "role": "assistant",1040 "content": "{\"action_type\": \"click\", \"x\": 558, \"y\": 360}"1041 }1042 ],1043 "images": [1044 "and_ctrl/out_episode_10432_step_001.png"1045 ]1046 },1047 {1048 "messages": [1049 {1050 "role": "system",1051 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1052 },1053 {1054 "role": "user",1055 "content": "<image>Open the Drive app."1056 },1057 {1058 "role": "assistant",1059 "content": "{\"action_type\": \"open_app\", \"app_name\": \"Drive\"}"1060 }1061 ],1062 "images": [1063 "and_ctrl/out_episode_12204_step_000.png"1064 ]1065 },1066 {1067 "messages": [1068 {1069 "role": "system",1070 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1071 },1072 {1073 "role": "user",1074 "content": "<image>Open Adani one app"1075 },1076 {1077 "role": "assistant",1078 "content": "{\"action_type\": \"open_app\", \"app_name\": \"Adani One\"}"1079 }1080 ],1081 "images": [1082 "and_ctrl/out_episode_15713_step_000.png"1083 ]1084 },1085 {1086 "messages": [1087 {1088 "role": "system",1089 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1090 },1091 {1092 "role": "user",1093 "content": "<image>Click on skip ad"1094 },1095 {1096 "role": "assistant",1097 "content": "{\"action_type\": \"wait\"}"1098 }1099 ],1100 "images": [1101 "and_ctrl/out_episode_19431_step_003.png"1102 ]1103 },1104 {1105 "messages": [1106 {1107 "role": "system",1108 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1109 },1110 {1111 "role": "user",1112 "content": "<image>go back"1113 },1114 {1115 "role": "assistant",1116 "content": "{\"action_type\": \"navigate_back\"}"1117 }1118 ],1119 "images": [1120 "and_ctrl/out_episode_16941_step_001.png"1121 ]1122 },1123 {1124 "messages": [1125 {1126 "role": "system",1127 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1128 },1129 {1130 "role": "user",1131 "content": "<image>Long press on the back space key of the key board to clear the title section."1132 },1133 {1134 "role": "assistant",1135 "content": "{\"action_type\": \"long_press\", \"x\": 994, \"y\": 2001}"1136 }1137 ],1138 "images": [1139 "and_ctrl/out_episode_18080_step_002.png"1140 ]1141 },1142 {1143 "messages": [1144 {1145 "role": "system",1146 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1147 },1148 {1149 "role": "user",1150 "content": "<image>Open the 1800 Flowers app."1151 },1152 {1153 "role": "assistant",1154 "content": "{\"action_type\": \"open_app\", \"app_name\": \"1800Flowers\"}"1155 }1156 ],1157 "images": [1158 "and_ctrl/out_episode_4677_step_000.png"1159 ]1160 },1161 {1162 "messages": [1163 {1164 "role": "system",1165 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1166 },1167 {1168 "role": "user",1169 "content": "<image>Long press on the first recording"1170 },1171 {1172 "role": "assistant",1173 "content": "{\"action_type\": \"long_press\", \"x\": 561, \"y\": 348}"1174 }1175 ],1176 "images": [1177 "and_ctrl/out_episode_7029_step_001.png"1178 ]1179 },1180 {1181 "messages": [1182 {1183 "role": "system",1184 "content": "You are a helpful assistant that can identify what action to perform on mobile UI Screenshot given the user instruction."1185 },1186 {1187 "role": "user",1188 "content": "<image>Open the Google Drive app"1189 },1190 {1191 "role": "assistant",1192 "content": "{\"action_type\": \"open_app\", \"app_name\": \"Drive\"}"1193 }1194 ],1195 "images": [1196 "and_ctrl/out_episode_926_step_000.png"1197 ]1198 },1199 {1200 "messages": [