OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0050345490833597",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1600,8 "w": 2560,9 "device_name": "Pixel Tablet"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Clock",15 "Amazon",16 "Chrome",17 "Instagram"18 ],19 "meta_task": "Plan a movie night by selecting a {} movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",20 "task": "Plan a movie night by selecting a horror movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",21 "instruction": "Organize a movie night by selecting a horror movie on Chrome, adding snacks to your Amazon cart, sending out invitations via Instagram, and setting a reminder on Clock. Make sure to share the details with Tzhau Jau."22 },23 "step_length": 33,24 "steps": [25 {26 "step": 0,27 "screenshot": "0050345490833597_0.png",28 "action": "CLICK",29 "info": [30 [31 143,32 71433 ],34 [35 143,36 71437 ]38 ],39 "ps": "[(143, 714)]"40 },41 {42 "step": 1,43 "screenshot": "0050345490833597_1.png",44 "action": "CLICK",45 "info": [46 [47 325,48 10849 ],50 [51 325,52 10853 ]54 ],55 "ps": "[(325, 108)]"56 },57 {58 "step": 2,59 "screenshot": "0050345490833597_2.png",60 "action": "TEXT",61 "info": "horror movies",62 "ps": ""63 },64 {65 "step": 3,66 "screenshot": "0050345490833597_3.png",67 "action": "CLICK",68 "info": [69 [70 853,71 69672 ],73 [74 853,75 69676 ]77 ],78 "ps": "[(853, 696)]"79 },80 {81 "step": 4,82 "screenshot": "0050345490833597_4.png",83 "action": "CLICK",84 "info": [85 [86 259,87 71388 ],89 [90 259,91 71392 ]93 ],94 "ps": "[(259, 713)]"95 },96 {97 "step": 5,98 "screenshot": "0050345490833597_5.png",99 "action": "SCROLL",100 "info": [101 [102 180,103 773104 ],105 [106 173,107 253108 ]109 ],110 "ps": "[(180, 773), (180, 772), (180, 768), (180, 766), (180, 763), (177, 759), (177, 756), (177, 753), (177, 751), (177, 747), (177, 744), (176, 741), (176, 736), (176, 731), (176, 728), (176, 724), (176, 723), (176, 723), (176, 722), (176, 719), (174, 719), (174, 714), (174, 711), (174, 706), (174, 703), (174, 702), (174, 697), (172, 692), (172, 687), (172, 682), (172, 674), (172, 666), (172, 656), (172, 643), (172, 633), (172, 621), (172, 611), (172, 598), (172, 586), (172, 573), (172, 563), (172, 553), (172, 541), (172, 531), (172, 518), (172, 508), (172, 496), (172, 488), (172, 479), (172, 469), (172, 461), (172, 453), (172, 443), (172, 437), (172, 427), (172, 421), (172, 413), (172, 408), (172, 406), (172, 398), (172, 393), (172, 388), (172, 383), (172, 378), (172, 373), (172, 364), (172, 357), (172, 349), (172, 343), (172, 338), (172, 333), (172, 328), (172, 323), (172, 319), (172, 313), (172, 308), (172, 303), (172, 299), (172, 294), (172, 289), (172, 284), (172, 282), (172, 281), (172, 278), (172, 274), (172, 272), (172, 268), (172, 267), (172, 263), (172, 259), (173, 258), (173, 256), (173, 253)]"111 },112 {113 "step": 6,114 "screenshot": "0050345490833597_6.png",115 "action": "CLICK",116 "info": "KEY_HOME",117 "ps": ""118 },119 {120 "step": 7,121 "screenshot": "0050345490833597_7.png",122 "action": "CLICK",123 "info": [124 [125 137,126 417127 ],128 [129 137,130 417131 ]132 ],133 "ps": "[(137, 417)]"134 },135 {136 "step": 8,137 "screenshot": "0050345490833597_8.png",138 "action": "CLICK",139 "info": [140 [141 75,142 67143 ],144 [145 75,146 67147 ]148 ],149 "ps": "[(75, 67)]"150 },151 {152 "step": 9,153 "screenshot": "0050345490833597_9.png",154 "action": "TEXT",155 "info": "snacks",156 "ps": ""157 },158 {159 "step": 10,160 "screenshot": "0050345490833597_10.png",161 "action": "CLICK",162 "info": [163 [164 896,165 684166 ],167 [168 896,169 684170 ]171 ],172 "ps": "[(896, 684)]"173 },174 {175 "step": 11,176 "screenshot": "0050345490833597_11.png",177 "action": "SCROLL",178 "info": [179 [180 371,181 782182 ],183 [184 382,185 107186 ]187 ],188 "ps": "[(371, 782), (371, 781), (371, 779), (371, 777), (371, 773), (371, 771), (371, 768), (371, 764), (371, 762), (371, 761), (371, 757), (371, 754), (371, 749), (371, 748), (371, 747), (371, 744), (371, 739), (371, 734), (371, 729), (371, 726), (371, 723), (371, 722), (371, 716), (371, 711), (371, 706), (371, 701), (371, 697), (371, 692), (371, 686), (371, 681), (371, 673), (371, 667), (371, 657), (371, 649), (371, 641), (371, 633), (371, 624), (371, 617), (371, 612), (371, 606), (371, 598), (371, 593), (371, 586), (371, 579), (371, 572), (371, 566), (371, 556), (371, 549), (371, 544), (371, 537), (371, 527), (371, 521), (371, 513), (371, 507), (371, 502), (371, 497), (371, 487), (371, 481), (373, 474), (373, 468), (373, 461), (373, 454), (373, 447), (373, 437), (373, 431), (373, 423), (373, 414), (373, 401), (373, 392), (373, 384), (373, 376), (373, 362), (373, 353), (373, 343), (375, 333), (375, 321), (375, 311), (375, 298), (375, 291), (376, 281), (376, 272), (376, 262), (376, 249), (376, 242), (376, 233), (376, 223), (376, 216), (376, 209), (377, 199), (377, 192), (378, 187), (378, 182), (378, 176), (378, 156), (380, 143), (380, 136), (380, 133), (380, 129), (380, 128), (380, 126), (380, 122), (380, 119), (382, 116), (382, 113), (382, 111), (382, 107)]"189 },190 {191 "step": 12,192 "screenshot": "0050345490833597_12.png",193 "action": "CLICK",194 "info": [195 [196 369,197 566198 ],199 [200 369,201 566202 ]203 ],204 "ps": "[(369, 566)]"205 },206 {207 "step": 13,208 "screenshot": "0050345490833597_13.png",209 "action": "SCROLL",210 "info": [211 [212 384,213 812214 ],215 [216 380,217 477218 ]219 ],220 "ps": "[(384, 812), (384, 809), (384, 807), (384, 803), (384, 798), (384, 793), (383, 791), (383, 787), (383, 784), (382, 779), (382, 774), (380, 771), (380, 763), (380, 758), (380, 753), (380, 748), (380, 739), (380, 734), (380, 732), (380, 728), (380, 723), (380, 717), (380, 707), (380, 697), (380, 688), (380, 674), (380, 662), (380, 656), (380, 646), (380, 636), (380, 623), (380, 614), (380, 598), (380, 591), (380, 582), (380, 569), (380, 562), (380, 553), (380, 546), (380, 538), (380, 532), (380, 524), (380, 522), (380, 518), (380, 516), (380, 512), (380, 509), (380, 507), (380, 503), (380, 501), (380, 497), (380, 494), (380, 492), (380, 488), (380, 486), (380, 482), (380, 479), (380, 477)]"221 },222 {223 "step": 14,224 "screenshot": "0050345490833597_14.png",225 "action": "SCROLL",226 "info": [227 [228 240,229 817230 ],231 [232 237,233 421234 ]235 ],236 "ps": "[(240, 817), (240, 816), (238, 812), (238, 809), (238, 806), (238, 803), (238, 801), (238, 797), (238, 794), (236, 791), (236, 786), (233, 781), (233, 778), (233, 772), (233, 767), (233, 761), (233, 756), (233, 754), (233, 752), (233, 747), (232, 741), (232, 738), (232, 736), (232, 731), (232, 726), (232, 721), (232, 716), (232, 708), (232, 701), (232, 696), (232, 691), (232, 686), (232, 681), (233, 673), (233, 668), (233, 663), (233, 658), (233, 652), (233, 647), (233, 642), (233, 637), (233, 629), (233, 623), (233, 618), (233, 614), (233, 612), (233, 607), (233, 602), (233, 598), (233, 597), (233, 593), (233, 588), (233, 584), (233, 583), (233, 581), (233, 578), (233, 573), (233, 568), (233, 564), (233, 562), (233, 557), (233, 552), (233, 548), (233, 546), (233, 541), (233, 538), (233, 534), (233, 532), (233, 528), (233, 526), (233, 523), (233, 519), (233, 517), (233, 513), (233, 512), (233, 508), (233, 504), (233, 499), (233, 497), (233, 492), (233, 488), (233, 486), (233, 483), (235, 479), (235, 477), (235, 473), (235, 471), (237, 467), (237, 464), (237, 461), (237, 456), (237, 453), (237, 451), (237, 447), (237, 444), (237, 441), (237, 439), (237, 436), (237, 432), (237, 429), (237, 426), (237, 423), (237, 421)]"237 },238 {239 "step": 15,240 "screenshot": "0050345490833597_15.png",241 "action": "CLICK",242 "info": [243 [244 100,245 588246 ],247 [248 100,249 588250 ]251 ],252 "ps": "[(100, 588)]"253 },254 {255 "step": 16,256 "screenshot": "0050345490833597_16.png",257 "action": "CLICK",258 "info": [259 [260 954,261 414262 ],263 [264 954,265 414266 ]267 ],268 "ps": "[(954, 414)]"269 },270 {271 "step": 17,272 "screenshot": "0050345490833597_17.png",273 "action": "CLICK",274 "info": "KEY_HOME",275 "ps": ""276 },277 {278 "step": 18,279 "screenshot": "0050345490833597_18.png",280 "action": "CLICK",281 "info": [282 [283 572,284 111285 ],286 [287 572,288 111289 ]290 ],291 "ps": "[(572, 111)]"292 },293 {294 "step": 19,295 "screenshot": "0050345490833597_19.png",296 "action": "CLICK",297 "info": [298 [299 14,300 487301 ],302 [303 14,304 487305 ]306 ],307 "ps": "[(14, 487)]"308 },309 {310 "step": 20,311 "screenshot": "0050345490833597_20.png",312 "action": "CLICK",313 "info": [314 [315 156,316 131317 ],318 [319 156,320 131321 ]322 ],323 "ps": "[(156, 131)]"324 },325 {326 "step": 21,327 "screenshot": "0050345490833597_21.png",328 "action": "CLICK",329 "info": [330 [331 35,332 213333 ],334 [335 35,336 213337 ]338 ],339 "ps": "[(35, 213)]"340 },341 {342 "step": 22,343 "screenshot": "0050345490833597_22.png",344 "action": "TEXT",345 "info": "What about a movie night this Saturday?",346 "ps": ""347 },348 {349 "step": 23,350 "screenshot": "0050345490833597_23.png",351 "action": "CLICK",352 "info": [353 [354 964,355 422356 ],357 [358 964,359 422360 ]361 ],362 "ps": "[(964, 422)]"363 },364 {365 "step": 24,366 "screenshot": "0050345490833597_24.png",367 "action": "CLICK",368 "info": "KEY_HOME",369 "ps": ""370 },371 {372 "step": 25,373 "screenshot": "0050345490833597_25.png",374 "action": "CLICK",375 "info": [376 [377 848,378 406379 ],380 [381 848,382 406383 ]384 ],385 "ps": "[(848, 406)]"386 },387 {388 "step": 26,389 "screenshot": "0050345490833597_26.png",390 "action": "CLICK",391 "info": [392 [393 771,394 887395 ],396 [397 771,398 887399 ]400 ],401 "ps": "[(771, 887)]"402 },403 {404 "step": 27,405 "screenshot": "0050345490833597_27.png",406 "action": "CLICK",407 "info": [408 [409 539,410 538411 ],412 [413 539,414 538415 ]416 ],417 "ps": "[(539, 538)]"418 },419 {420 "step": 28,421 "screenshot": "0050345490833597_28.png",422 "action": "CLICK",423 "info": [424 [425 690,426 689427 ],428 [429 690,430 689431 ]432 ],433 "ps": "[(690, 689)]"434 },435 {436 "step": 29,437 "screenshot": "0050345490833597_29.png",438 "action": "CLICK",439 "info": [440 [441 666,442 419443 ],444 [445 666,446 419447 ]448 ],449 "ps": "[(666, 419)]"450 },451 {452 "step": 30,453 "screenshot": "0050345490833597_30.png",454 "action": "CLICK",455 "info": [456 [457 681,458 543459 ],460 [461 681,462 543463 ]464 ],465 "ps": "[(681, 543)]"466 },467 {468 "step": 31,469 "screenshot": "0050345490833597_31.png",470 "action": "CLICK",471 "info": [472 [473 676,474 718475 ],476 [477 676,478 718479 ]480 ],481 "ps": "[(676, 718)]"482 },483 {484 "step": 32,485 "screenshot": "0050345490833597_32.png",486 "action": "COMPLETE",487 "info": "",488 "ps": "Tzhau Jau"489 }490 ]491}