OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "1255818892684321",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2992,8 "w": 1344,9 "device_name": "Pixel 8 Pro"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Microsoft to do",15 "Opera",16 "Ebay"17 ],18 "meta_task": "Find a recipe for {}, create a shopping list of main ingredients, and add to the cart.",19 "task": "Find a recipe for Chocolate chip cookies, create a shopping list of main ingredients, and add to the cart.",20 "instruction": "Utilize Opera to search for a Chocolate chip cookies recipe, then compile a shopping list of the necessary main ingredients using Microsoft To Do, and finally, add these items to the cart on Ebay."21 },22 "step_length": 23,23 "steps": [24 {25 "step": 0,26 "screenshot": "1255818892684321_0.png",27 "action": "CLICK",28 "info": [29 [30 869,31 84332 ],33 [34 869,35 84336 ]37 ],38 "ps": "[(869, 843)]"39 },40 {41 "step": 1,42 "screenshot": "1255818892684321_1.png",43 "action": "CLICK",44 "info": "KEY_HOME",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "1255818892684321_2.png",50 "action": "CLICK",51 "info": [52 [53 382,54 38455 ],56 [57 382,58 38459 ]60 ],61 "ps": "[(382, 384)]"62 },63 {64 "step": 3,65 "screenshot": "1255818892684321_3.png",66 "action": "CLICK",67 "info": [68 [69 659,70 57371 ],72 [73 659,74 57375 ]76 ],77 "ps": "[(659, 573)]"78 },79 {80 "step": 4,81 "screenshot": "1255818892684321_4.png",82 "action": "CLICK",83 "info": [84 [85 900,86 92587 ],88 [89 900,90 92591 ]92 ],93 "ps": "[(900, 925)]"94 },95 {96 "step": 5,97 "screenshot": "1255818892684321_5.png",98 "action": "TEXT",99 "info": "Chocolate chip cookies: 150g salted butter, softened 80g light brown muscovado sugar, 80g granulated sugar, 2 tsp vanilla extract, 1 large egg, 225g plain flour, 1/2 tsp bicarbonate of soda, 1/4 tsp salt, 200g plain chocolate chips or chunks",100 "ps": ""101 },102 {103 "step": 6,104 "screenshot": "1255818892684321_6.png",105 "action": "CLICK",106 "info": [107 [108 902,109 547110 ],111 [112 902,113 547114 ]115 ],116 "ps": "[(902, 547)]"117 },118 {119 "step": 7,120 "screenshot": "1255818892684321_7.png",121 "action": "CLICK",122 "info": "KEY_HOME",123 "ps": ""124 },125 {126 "step": 8,127 "screenshot": "1255818892684321_8.png",128 "action": "CLICK",129 "info": [130 [131 165,132 527133 ],134 [135 165,136 527137 ]138 ],139 "ps": "[(165, 527)]"140 },141 {142 "step": 9,143 "screenshot": "1255818892684321_9.png",144 "action": "CLICK",145 "info": [146 [147 393,148 152149 ],150 [151 393,152 152153 ]154 ],155 "ps": "[(393, 152)]"156 },157 {158 "step": 10,159 "screenshot": "1255818892684321_10.png",160 "action": "CLICK",161 "info": [162 [163 429,164 207165 ],166 [167 429,168 207169 ]170 ],171 "ps": "[(429, 207)]"172 },173 {174 "step": 11,175 "screenshot": "1255818892684321_11.png",176 "action": "CLICK",177 "info": [178 [179 449,180 426181 ],182 [183 449,184 426185 ]186 ],187 "ps": "[(449, 426)]"188 },189 {190 "step": 12,191 "screenshot": "1255818892684321_12.png",192 "action": "CLICK",193 "info": [194 [195 496,196 429197 ],198 [199 496,200 429201 ]202 ],203 "ps": "[(496, 429)]"204 },205 {206 "step": 13,207 "screenshot": "1255818892684321_13.png",208 "action": "SCROLL",209 "info": [210 [211 494,212 850213 ],214 [215 531,216 542217 ]218 ],219 "ps": "[(494, 850), (494, 849), (494, 847), (494, 846), (496, 844), (496, 842), (496, 841), (496, 839), (498, 839), (498, 838), (498, 836), (498, 834), (498, 833), (498, 831), (502, 830), (502, 828), (502, 826), (502, 825), (502, 823), (502, 821), (505, 820), (505, 818), (505, 817), (505, 815), (505, 813), (505, 812), (509, 810), (509, 808), (509, 807), (509, 805), (509, 804), (509, 802), (509, 800), (509, 799), (509, 797), (509, 795), (509, 794), (509, 792), (509, 791), (514, 789), (514, 787), (514, 786), (514, 784), (514, 782), (514, 781), (514, 779), (514, 778), (514, 776), (514, 774), (514, 773), (516, 771), (516, 770), (516, 768), (516, 766), (516, 765), (516, 763), (516, 761), (516, 760), (516, 758), (516, 757), (516, 754), (516, 752), (516, 751), (516, 749), (516, 747), (516, 746), (516, 744), (516, 743), (516, 741), (516, 739), (516, 738), (516, 736), (516, 734), (516, 733), (516, 731), (516, 730), (516, 728), (516, 726), (516, 725), (516, 723), (516, 722), (520, 720), (520, 718), (520, 717), (520, 715), (520, 713), (520, 712), (520, 710), (520, 709), (520, 707), (520, 705), (520, 704), (520, 702), (520, 700), (520, 699), (520, 697), (520, 696), (520, 694), (520, 692), (520, 691), (520, 689), (520, 687), (520, 686), (520, 684), (520, 683), (520, 681), (520, 679), (520, 676), (520, 674), (520, 673), (520, 671), (520, 669), (520, 668), (520, 666), (520, 665), (520, 663), (520, 661), (520, 660), (520, 658), (520, 656), (520, 655), (520, 653), (520, 652), (520, 648), (520, 644), (520, 641), (525, 637), (525, 635), (525, 634), (525, 632), (525, 630), (525, 629), (525, 627), (525, 626), (525, 622), (525, 619), (525, 615), (525, 613), (525, 612), (525, 610), (525, 608), (525, 607), (525, 605), (525, 604), (525, 602), (525, 600), (525, 599), (525, 597), (525, 595), (525, 594), (525, 592), (525, 591), (525, 589), (525, 587), (525, 586), (525, 584), (525, 582), (525, 581), (525, 579), (527, 578), (527, 576), (527, 574), (527, 573), (527, 571), (527, 569), (527, 568), (527, 566), (527, 565), (527, 563), (527, 561), (527, 560), (527, 558), (527, 556), (527, 555), (527, 553), (527, 552), (531, 550), (531, 548), (531, 547), (531, 545), (531, 544), (531, 542)]"220 },221 {222 "step": 14,223 "screenshot": "1255818892684321_14.png",224 "action": "CLICK",225 "info": [226 [227 549,228 822229 ],230 [231 549,232 822233 ]234 ],235 "ps": "[(549, 822)]"236 },237 {238 "step": 15,239 "screenshot": "1255818892684321_15.png",240 "action": "CLICK",241 "info": [242 [243 560,244 491245 ],246 [247 560,248 491249 ]250 ],251 "ps": "[(560, 491)]"252 },253 {254 "step": 16,255 "screenshot": "1255818892684321_16.png",256 "action": "CLICK",257 "info": [258 [259 607,260 92261 ],262 [263 607,264 92265 ]266 ],267 "ps": "[(607, 92)]"268 },269 {270 "step": 17,271 "screenshot": "1255818892684321_17.png",272 "action": "CLICK",273 "info": [274 [275 810,276 77277 ],278 [279 810,280 77281 ]282 ],283 "ps": "[(810, 77)]"284 },285 {286 "step": 18,287 "screenshot": "1255818892684321_18.png",288 "action": "TEXT",289 "info": "chocolate chips",290 "ps": ""291 },292 {293 "step": 19,294 "screenshot": "1255818892684321_19.png",295 "action": "CLICK",296 "info": [297 [298 906,299 910300 ],301 [302 906,303 908304 ]305 ],306 "ps": "[(906, 910), (906, 908)]"307 },308 {309 "step": 20,310 "screenshot": "1255818892684321_20.png",311 "action": "CLICK",312 "info": [313 [314 576,315 403316 ],317 [318 576,319 403320 ]321 ],322 "ps": "[(576, 403)]"323 },324 {325 "step": 21,326 "screenshot": "1255818892684321_21.png",327 "action": "CLICK",328 "info": [329 [330 264,331 921332 ],333 [334 264,335 921336 ]337 ],338 "ps": "[(264, 921)]"339 },340 {341 "step": 22,342 "screenshot": "1255818892684321_22.png",343 "action": "COMPLETE",344 "info": "",345 "ps": ""346 }347 ]348}