OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0177347764055831",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1280,8 "w": 720,9 "device_name": "Small Phone"10 },11 "task_info": {12 "category": "Information_Management",13 "app": [14 "Weather & Radar",15 "Microsoft word"16 ],17 "meta_task": "Look up the weather forecast in {} tomorrow and plan a to-do list accordingly.",18 "task": "Look up the weather forecast in Cape Town tomorrow and plan a to-do list accordingly.",19 "instruction": "Using Weather & Radar, check the weather forecast for Cape Town tomorrow. Based on the forecast, create a to-do list in Microsoft Word."20 },21 "step_length": 24,22 "steps": [23 {24 "step": 0,25 "screenshot": "0177347764055831_0.png",26 "action": "CLICK",27 "info": "KEY_APPSELECT",28 "ps": ""29 },30 {31 "step": 1,32 "screenshot": "0177347764055831_1.png",33 "action": "SCROLL",34 "info": [35 [36 511,37 65338 ],39 [40 522,41 31442 ]43 ],44 "ps": "[(511, 653), (513, 653), (513, 651), (513, 645), (513, 632), (508, 614), (505, 592), (505, 567), (505, 539), (505, 514), (505, 489), (505, 467), (505, 448), (505, 429), (505, 414), (505, 395), (505, 376), (511, 359), (513, 346), (519, 329), (522, 314)]"45 },46 {47 "step": 2,48 "screenshot": "0177347764055831_2.png",49 "action": "SCROLL",50 "info": [51 [52 491,53 66854 ],55 [56 527,57 058 ]59 ],60 "ps": "[(491, 668), (494, 667), (497, 662), (497, 648), (502, 620), (508, 576), (513, 510), (519, 432), (527, 337), (527, 229), (527, 115), (527, 0)]"61 },62 {63 "step": 3,64 "screenshot": "0177347764055831_3.png",65 "action": "SCROLL",66 "info": [67 [68 624,69 71270 ],71 [72 716,73 074 ]75 ],76 "ps": "[(624, 712), (622, 712), (622, 709), (619, 701), (613, 682), (613, 654), (613, 607), (613, 518), (613, 410), (627, 293), (652, 171), (683, 53), (716, 0)]"77 },78 {79 "step": 4,80 "screenshot": "0177347764055831_4.png",81 "action": "CLICK",82 "info": "KEY_HOME",83 "ps": ""84 },85 {86 "step": 5,87 "screenshot": "0177347764055831_5.png",88 "action": "CLICK",89 "info": [90 [91 716,92 48493 ],94 [95 716,96 48497 ]98 ],99 "ps": "[(716, 484)]"100 },101 {102 "step": 6,103 "screenshot": "0177347764055831_6.png",104 "action": "CLICK",105 "info": [106 [107 922,108 79109 ],110 [111 922,112 79113 ]114 ],115 "ps": "[(922, 79)]"116 },117 {118 "step": 7,119 "screenshot": "0177347764055831_7.png",120 "action": "CLICK",121 "info": [122 [123 97,124 45125 ],126 [127 97,128 45129 ]130 ],131 "ps": "[(97, 45)]"132 },133 {134 "step": 8,135 "screenshot": "0177347764055831_8.png",136 "action": "CLICK",137 "info": [138 [139 472,140 190141 ],142 [143 472,144 190145 ]146 ],147 "ps": "[(472, 190)]"148 },149 {150 "step": 9,151 "screenshot": "0177347764055831_9.png",152 "action": "TEXT",153 "info": "Cape town",154 "ps": ""155 },156 {157 "step": 10,158 "screenshot": "0177347764055831_10.png",159 "action": "CLICK",160 "info": [161 [162 436,163 299164 ],165 [166 436,167 299168 ]169 ],170 "ps": "[(436, 299)]"171 },172 {173 "step": 11,174 "screenshot": "0177347764055831_11.png",175 "action": "SCROLL",176 "info": [177 [178 583,179 779180 ],181 [182 502,183 157184 ]185 ],186 "ps": "[(583, 779), (583, 776), (583, 773), (583, 765), (583, 759), (583, 751), (583, 742), (583, 732), (583, 723), (583, 710), (583, 706), (583, 703), (583, 695), (583, 684), (583, 665), (580, 645), (574, 623), (574, 604), (572, 585), (572, 570), (569, 556), (563, 546), (561, 535), (558, 523), (558, 514), (552, 503), (549, 490), (544, 476), (544, 467), (544, 457), (544, 448), (541, 437), (538, 428), (536, 420), (536, 410), (536, 401), (530, 390), (527, 381), (527, 373), (527, 367), (524, 357), (524, 356), (522, 353), (519, 348), (519, 345), (519, 340), (519, 334), (519, 329), (519, 326), (516, 321), (516, 318), (516, 315), (513, 314), (513, 312), (513, 310), (511, 307), (511, 304), (511, 303), (511, 299), (511, 296), (511, 295), (508, 295), (508, 290), (508, 287), (508, 282), (508, 279), (508, 273), (508, 271), (508, 265), (508, 259), (508, 251), (508, 245), (508, 243), (508, 240), (508, 237), (508, 234), (508, 229), (508, 223), (508, 215), (508, 210), (508, 209), (508, 207), (508, 206), (508, 204), (508, 203), (508, 201), (508, 199), (508, 198), (508, 196), (508, 195), (508, 193), (508, 192), (508, 190), (508, 189), (508, 187), (508, 185), (508, 184), (505, 182), (505, 178), (505, 176), (505, 174), (505, 173), (505, 171), (505, 170), (505, 167), (505, 165), (505, 164), (502, 162), (502, 160), (502, 159), (502, 157)]"187 },188 {189 "step": 12,190 "screenshot": "0177347764055831_12.png",191 "action": "SCROLL",192 "info": [193 [194 572,195 837196 ],197 [198 516,199 173200 ]201 ],202 "ps": "[(572, 837), (563, 728), (558, 709), (555, 690), (549, 668), (547, 646), (547, 628), (547, 609), (541, 592), (541, 579), (541, 567), (538, 556), (538, 546), (538, 534), (538, 524), (538, 515), (538, 506), (538, 495), (536, 485), (530, 474), (530, 468), (530, 460), (530, 454), (530, 446), (527, 442), (527, 439), (527, 434), (524, 426), (524, 420), (524, 410), (522, 403), (522, 396), (522, 387), (522, 378), (522, 370), (522, 364), (522, 356), (522, 349), (522, 342), (522, 339), (522, 335), (522, 331), (522, 326), (522, 324), (522, 321), (522, 320), (522, 318), (522, 317), (522, 315), (522, 314), (522, 312), (522, 310), (522, 309), (522, 306), (522, 303), (522, 298), (522, 295), (522, 289), (522, 284), (522, 281), (522, 276), (522, 274), (519, 270), (519, 267), (519, 265), (519, 259), (519, 256), (519, 253), (519, 248), (519, 246), (519, 245), (519, 243), (519, 239), (519, 235), (519, 234), (519, 232), (519, 228), (519, 226), (519, 224), (519, 223), (519, 220), (519, 218), (519, 217), (519, 215), (519, 214), (519, 212), (519, 210), (519, 209), (519, 207), (519, 204), (519, 198), (519, 196), (519, 195), (519, 193), (519, 190), (516, 189), (516, 185), (516, 184), (516, 182), (516, 181), (516, 179), (516, 178), (516, 176), (516, 174), (516, 173)]"203 },204 {205 "step": 13,206 "screenshot": "0177347764055831_13.png",207 "action": "SCROLL",208 "info": [209 [210 544,211 864212 ],213 [214 555,215 145216 ]217 ],218 "ps": "[(544, 864), (519, 701), (513, 682), (511, 664), (505, 640), (505, 621), (505, 603), (505, 587), (505, 574), (505, 562), (505, 553), (505, 543), (505, 540), (505, 539), (505, 534), (505, 531), (502, 526), (502, 523), (502, 518), (499, 512), (499, 507), (499, 506), (499, 503), (499, 501), (499, 495), (499, 492), (497, 485), (497, 482), (497, 478), (497, 473), (497, 470), (497, 468), (497, 465), (497, 462), (497, 459), (497, 456), (497, 453), (497, 449), (497, 445), (497, 440), (497, 437), (497, 432), (497, 424), (499, 420), (499, 417), (499, 412), (499, 407), (499, 401), (499, 396), (499, 392), (499, 387), (499, 379), (502, 370), (508, 360), (508, 349), (511, 340), (511, 332), (513, 326), (516, 315), (522, 299), (524, 284), (530, 270), (533, 260), (533, 251), (536, 240), (541, 231), (541, 220), (547, 217), (547, 212), (549, 204), (549, 198), (549, 190), (552, 184), (552, 176), (555, 170), (555, 162), (555, 159), (555, 156), (555, 154), (555, 153), (555, 151), (555, 149), (555, 148), (555, 145)]"219 },220 {221 "step": 14,222 "screenshot": "0177347764055831_14.png",223 "action": "SCROLL",224 "info": [225 [226 508,227 868228 ],229 [230 583,231 251232 ]233 ],234 "ps": "[(508, 868), (511, 868), (513, 868), (513, 867), (513, 860), (513, 854), (516, 846), (516, 837), (516, 832), (516, 828), (516, 818), (522, 801), (524, 785), (524, 770), (524, 757), (530, 745), (533, 731), (533, 721), (536, 710), (536, 701), (541, 692), (541, 682), (541, 674), (541, 668), (544, 660), (544, 653), (544, 648), (544, 645), (547, 640), (547, 637), (547, 634), (547, 631), (547, 626), (549, 623), (549, 618), (549, 614), (549, 612), (549, 607), (552, 604), (552, 601), (552, 596), (552, 592), (552, 589), (552, 585), (552, 582), (552, 574), (552, 570), (552, 567), (552, 560), (552, 553), (552, 548), (552, 543), (552, 539), (552, 532), (552, 528), (552, 523), (552, 515), (552, 509), (555, 501), (555, 496), (555, 493), (555, 489), (555, 484), (555, 479), (555, 476), (555, 468), (555, 464), (555, 460), (558, 456), (561, 448), (561, 442), (561, 434), (563, 424), (569, 417), (569, 407), (569, 401), (569, 392), (569, 387), (569, 381), (569, 376), (569, 370), (572, 362), (572, 359), (572, 354), (572, 349), (574, 343), (574, 342), (574, 339), (574, 335), (574, 332), (574, 331), (574, 329), (577, 328), (577, 324), (577, 323), (577, 320), (577, 318), (580, 314), (580, 310), (583, 307), (583, 303), (583, 299), (583, 296), (583, 293), (583, 290), (583, 285), (583, 282), (583, 279), (583, 274), (583, 273), (583, 270), (583, 267), (583, 265), (583, 262), (583, 260), (583, 259), (583, 257), (583, 254), (583, 253), (583, 251)]"235 },236 {237 "step": 15,238 "screenshot": "0177347764055831_15.png",239 "action": "CLICK",240 "info": [241 [242 341,243 859244 ],245 [246 341,247 859248 ]249 ],250 "ps": "[(341, 859)]"251 },252 {253 "step": 16,254 "screenshot": "0177347764055831_16.png",255 "action": "SCROLL",256 "info": [257 [258 636,259 853260 ],261 [262 616,263 507264 ]265 ],266 "ps": "[(636, 853), (638, 853), (638, 849), (638, 842), (638, 835), (638, 828), (638, 818), (638, 806), (638, 793), (638, 778), (638, 759), (638, 743), (638, 724), (638, 709), (638, 693), (638, 678), (638, 660), (638, 645), (638, 632), (638, 620), (638, 607), (638, 604), (638, 601), (638, 596), (636, 595), (636, 590), (636, 585), (636, 582), (633, 578), (633, 574), (633, 570), (630, 565), (627, 564), (624, 559), (624, 556), (622, 554), (622, 549), (622, 546), (622, 545), (622, 543), (622, 540), (622, 537), (619, 535), (619, 534), (619, 531), (619, 528), (619, 523), (619, 521), (619, 520), (616, 518), (616, 517), (616, 515), (616, 514), (616, 512), (616, 510), (616, 509), (616, 507)]"267 },268 {269 "step": 17,270 "screenshot": "0177347764055831_17.png",271 "action": "CLICK",272 "info": "KEY_HOME",273 "ps": ""274 },275 {276 "step": 18,277 "screenshot": "0177347764055831_18.png",278 "action": "CLICK",279 "info": [280 [281 713,282 615283 ],284 [285 713,286 615287 ]288 ],289 "ps": "[(713, 615)]"290 },291 {292 "step": 19,293 "screenshot": "0177347764055831_19.png",294 "action": "CLICK",295 "info": [296 [297 905,298 73299 ],300 [301 905,302 73303 ]304 ],305 "ps": "[(905, 73)]"306 },307 {308 "step": 20,309 "screenshot": "0177347764055831_20.png",310 "action": "CLICK",311 "info": [312 [313 208,314 434315 ],316 [317 208,318 434319 ]320 ],321 "ps": "[(208, 434)]"322 },323 {324 "step": 21,325 "screenshot": "0177347764055831_21.png",326 "action": "TEXT",327 "info": "Cape town,tomorrow:cloudy Todolist: plan a trip in Cape Town and invite friends.",328 "ps": ""329 },330 {331 "step": 22,332 "screenshot": "0177347764055831_22.png",333 "action": "SCROLL",334 "info": [335 [336 63,337 353338 ],339 [340 561,341 359342 ]343 ],344 "ps": "[(63, 353), (63, 354), (66, 356), (72, 356), (86, 356), (97, 356), (113, 356), (130, 356), (138, 356), (149, 356), (169, 356), (202, 356), (241, 356), (280, 356), (349, 357), (383, 360), (416, 362), (449, 362), (483, 362), (511, 362), (533, 362), (552, 362), (561, 362), (563, 362), (563, 360), (561, 359)]"345 },346 {347 "step": 23,348 "screenshot": "0177347764055831_23.png",349 "action": "COMPLETE",350 "info": "",351 "ps": ""352 }353 ]354}