OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0084442481728114",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1600,8 "w": 2560,9 "device_name": "Pixel Tablet"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Chrome",15 "Clock",16 "Ebay",17 "Facebook"18 ],19 "meta_task": "Plan a movie night by selecting a {} movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",20 "task": "Plan a movie night by selecting a thriller movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",21 "instruction": "Organize a movie night by selecting a thriller film using Chrome, adding snacks to your cart on Ebay, sending out invitations via Facebook, and setting a reminder on your Clock app. Remember to invite Victor James to join the event."22 },23 "step_length": 39,24 "steps": [25 {26 "step": 0,27 "screenshot": "0084442481728114_0.png",28 "action": "CLICK",29 "info": [30 [31 134,32 69633 ],34 [35 134,36 69637 ]38 ],39 "ps": "[(134, 696)]"40 },41 {42 "step": 1,43 "screenshot": "0084442481728114_1.png",44 "action": "CLICK",45 "info": [46 [47 291,48 38649 ],50 [51 291,52 38653 ]54 ],55 "ps": "[(291, 386)]"56 },57 {58 "step": 2,59 "screenshot": "0084442481728114_2.png",60 "action": "TEXT",61 "info": "thriller movies",62 "ps": ""63 },64 {65 "step": 3,66 "screenshot": "0084442481728114_3.png",67 "action": "CLICK",68 "info": [69 [70 850,71 69872 ],73 [74 852,75 69676 ]77 ],78 "ps": "[(850, 698), (852, 696)]"79 },80 {81 "step": 4,82 "screenshot": "0084442481728114_4.png",83 "action": "CLICK",84 "info": [85 [86 258,87 80888 ],89 [90 258,91 80892 ]93 ],94 "ps": "[(258, 808)]"95 },96 {97 "step": 5,98 "screenshot": "0084442481728114_5.png",99 "action": "SCROLL",100 "info": [101 [102 331,103 853104 ],105 [106 333,107 396108 ]109 ],110 "ps": "[(331, 853), (331, 852), (331, 849), (331, 846), (329, 843), (329, 839), (329, 834), (329, 829), (329, 827), (329, 822), (329, 817), (329, 812), (329, 807), (329, 799), (329, 791), (329, 781), (329, 773), (329, 764), (329, 754), (329, 747), (329, 738), (329, 722), (329, 709), (329, 699), (329, 687), (329, 674), (329, 664), (329, 652), (329, 639), (329, 629), (329, 621), (329, 604), (329, 586), (330, 572), (330, 559), (331, 549), (331, 537), (331, 527), (331, 514), (331, 507), (331, 498), (331, 491), (331, 482), (331, 474), (331, 472), (331, 471), (331, 468), (331, 463), (331, 458), (331, 454), (331, 452), (331, 448), (331, 446), (331, 443), (331, 439), (331, 434), (331, 432), (331, 428), (331, 426), (331, 423), (331, 419), (331, 417), (331, 413), (331, 411), (333, 407), (333, 404), (333, 401), (333, 398), (333, 396)]"111 },112 {113 "step": 6,114 "screenshot": "0084442481728114_6.png",115 "action": "CLICK",116 "info": "KEY_HOME",117 "ps": ""118 },119 {120 "step": 7,121 "screenshot": "0084442481728114_7.png",122 "action": "CLICK",123 "info": [124 [125 273,126 417127 ],128 [129 273,130 417131 ]132 ],133 "ps": "[(273, 417)]"134 },135 {136 "step": 8,137 "screenshot": "0084442481728114_8.png",138 "action": "CLICK",139 "info": [140 [141 235,142 149143 ],144 [145 235,146 149147 ]148 ],149 "ps": "[(235, 149)]"150 },151 {152 "step": 9,153 "screenshot": "0084442481728114_9.png",154 "action": "TEXT",155 "info": "snacks",156 "ps": ""157 },158 {159 "step": 10,160 "screenshot": "0084442481728114_10.png",161 "action": "CLICK",162 "info": [163 [164 894,165 696166 ],167 [168 894,169 696170 ]171 ],172 "ps": "[(894, 696)]"173 },174 {175 "step": 11,176 "screenshot": "0084442481728114_11.png",177 "action": "CLICK",178 "info": [179 [180 364,181 546182 ],183 [184 364,185 546186 ]187 ],188 "ps": "[(364, 546)]"189 },190 {191 "step": 12,192 "screenshot": "0084442481728114_12.png",193 "action": "CLICK",194 "info": [195 [196 669,197 531198 ],199 [200 669,201 531202 ]203 ],204 "ps": "[(669, 531)]"205 },206 {207 "step": 13,208 "screenshot": "0084442481728114_13.png",209 "action": "CLICK",210 "info": [211 [212 351,213 536214 ],215 [216 351,217 536218 ]219 ],220 "ps": "[(351, 536)]"221 },222 {223 "step": 14,224 "screenshot": "0084442481728114_14.png",225 "action": "SCROLL",226 "info": [227 [228 297,229 866230 ],231 [232 296,233 491234 ]235 ],236 "ps": "[(297, 866), (297, 862), (297, 859), (297, 857), (297, 853), (297, 852), (297, 849), (297, 847), (297, 844), (297, 842), (297, 839), (297, 834), (297, 831), (297, 828), (297, 821), (297, 814), (297, 807), (297, 797), (297, 784), (297, 777), (297, 771), (297, 766), (297, 761), (297, 756), (297, 748), (297, 739), (297, 732), (297, 724), (297, 716), (297, 706), (297, 697), (297, 689), (297, 679), (297, 671), (297, 663), (297, 652), (297, 644), (297, 639), (297, 634), (297, 633), (297, 631), (297, 626), (297, 621), (297, 613), (297, 606), (297, 601), (297, 596), (297, 593), (297, 588), (297, 583), (297, 578), (297, 576), (297, 571), (297, 566), (297, 561), (297, 557), (297, 552), (297, 549), (297, 548), (297, 544), (297, 542), (297, 541), (297, 538), (297, 537), (297, 533), (297, 532), (297, 529), (297, 528), (297, 526), (297, 521), (297, 519), (297, 518), (297, 517), (297, 513), (297, 511), (297, 509), (297, 508), (297, 504), (297, 502), (297, 499), (297, 498), (297, 497), (297, 493), (296, 491)]"237 },238 {239 "step": 15,240 "screenshot": "0084442481728114_15.png",241 "action": "SCROLL",242 "info": [243 [244 254,245 708246 ],247 [248 248,249 314250 ]251 ],252 "ps": "[(254, 708), (254, 706), (254, 703), (253, 699), (253, 697), (251, 693), (251, 691), (251, 687), (251, 684), (251, 682), (251, 677), (248, 671), (248, 666), (248, 661), (248, 656), (248, 651), (248, 646), (248, 643), (248, 638), (248, 633), (248, 632), (248, 629), (248, 624), (248, 617), (248, 609), (248, 601), (248, 591), (248, 586), (248, 584), (248, 579), (248, 574), (248, 568), (248, 558), (248, 549), (248, 539), (248, 529), (248, 519), (248, 513), (248, 506), (248, 497), (248, 489), (249, 484), (249, 476), (249, 468), (249, 463), (249, 458), (249, 456), (249, 451), (249, 447), (249, 442), (249, 439), (249, 436), (249, 433), (248, 428), (248, 424), (248, 422), (248, 419), (248, 416), (248, 413), (248, 409), (248, 408), (248, 407), (248, 404), (248, 402), (248, 398), (248, 396), (248, 392), (248, 389), (248, 384), (248, 379), (248, 377), (248, 373), (248, 372), (248, 369), (248, 366), (248, 363), (248, 361), (248, 357), (248, 354), (248, 349), (248, 346), (248, 343), (248, 341), (248, 336), (248, 332), (248, 329), (248, 326), (248, 323), (248, 321), (248, 317), (248, 314)]"253 },254 {255 "step": 16,256 "screenshot": "0084442481728114_16.png",257 "action": "CLICK",258 "info": [259 [260 474,261 607262 ],263 [264 474,265 607266 ]267 ],268 "ps": "[(474, 607)]"269 },270 {271 "step": 17,272 "screenshot": "0084442481728114_17.png",273 "action": "CLICK",274 "info": [275 [276 668,277 564278 ],279 [280 668,281 564282 ]283 ],284 "ps": "[(668, 564)]"285 },286 {287 "step": 18,288 "screenshot": "0084442481728114_18.png",289 "action": "CLICK",290 "info": "KEY_HOME",291 "ps": ""292 },293 {294 "step": 19,295 "screenshot": "0084442481728114_19.png",296 "action": "CLICK",297 "info": [298 [299 426,300 127301 ],302 [303 426,304 127305 ]306 ],307 "ps": "[(426, 127)]"308 },309 {310 "step": 20,311 "screenshot": "0084442481728114_20.png",312 "action": "CLICK",313 "info": [314 [315 390,316 112317 ],318 [319 390,320 112321 ]322 ],323 "ps": "[(390, 112)]"324 },325 {326 "step": 21,327 "screenshot": "0084442481728114_21.png",328 "action": "CLICK",329 "info": [330 [331 431,332 158333 ],334 [335 431,336 158337 ]338 ],339 "ps": "[(431, 158)]"340 },341 {342 "step": 22,343 "screenshot": "0084442481728114_22.png",344 "action": "CLICK",345 "info": [346 [347 115,348 262349 ],350 [351 115,352 262353 ]354 ],355 "ps": "[(115, 262)]"356 },357 {358 "step": 23,359 "screenshot": "0084442481728114_23.png",360 "action": "SCROLL",361 "info": [362 [363 407,364 768365 ],366 [367 391,368 328369 ]370 ],371 "ps": "[(407, 768), (407, 767), (407, 764), (407, 759), (405, 757), (404, 753), (404, 748), (403, 744), (403, 742), (401, 737), (399, 729), (398, 719), (398, 707), (395, 704), (394, 697), (391, 684), (390, 674), (390, 667), (390, 654), (390, 636), (390, 617), (390, 602), (390, 586), (390, 564), (390, 546), (391, 534), (391, 522), (391, 509), (391, 497), (391, 487), (391, 471), (391, 458), (391, 446), (391, 429), (391, 417), (391, 407), (391, 394), (391, 384), (391, 378), (391, 368), (391, 362), (391, 354), (391, 352), (391, 351), (391, 347), (391, 344), (391, 342), (391, 338), (391, 337), (391, 334), (391, 331), (391, 328)]"372 },373 {374 "step": 24,375 "screenshot": "0084442481728114_24.png",376 "action": "CLICK",377 "info": [378 [379 498,380 721381 ],382 [383 498,384 721385 ]386 ],387 "ps": "[(498, 721)]"388 },389 {390 "step": 25,391 "screenshot": "0084442481728114_25.png",392 "action": "CLICK",393 "info": [394 [395 185,396 911397 ],398 [399 185,400 911401 ]402 ],403 "ps": "[(185, 911)]"404 },405 {406 "step": 26,407 "screenshot": "0084442481728114_26.png",408 "action": "TEXT",409 "info": "I'd like to invite you for a movie night this Sunday",410 "ps": ""411 },412 {413 "step": 27,414 "screenshot": "0084442481728114_27.png",415 "action": "CLICK",416 "info": [417 [418 971,419 426420 ],421 [422 971,423 426424 ]425 ],426 "ps": "[(971, 426)]"427 },428 {429 "step": 28,430 "screenshot": "0084442481728114_28.png",431 "action": "CLICK",432 "info": "KEY_HOME",433 "ps": ""434 },435 {436 "step": 29,437 "screenshot": "0084442481728114_29.png",438 "action": "CLICK",439 "info": [440 [441 853,442 424443 ],444 [445 853,446 424447 ]448 ],449 "ps": "[(853, 424)]"450 },451 {452 "step": 30,453 "screenshot": "0084442481728114_30.png",454 "action": "CLICK",455 "info": [456 [457 747,458 884459 ],460 [461 747,462 884463 ]464 ],465 "ps": "[(747, 884)]"466 },467 {468 "step": 31,469 "screenshot": "0084442481728114_31.png",470 "action": "CLICK",471 "info": [472 [473 522,474 448475 ],476 [477 522,478 448479 ]480 ],481 "ps": "[(522, 448)]"482 },483 {484 "step": 32,485 "screenshot": "0084442481728114_32.png",486 "action": "CLICK",487 "info": [488 [489 677,490 538491 ],492 [493 677,494 538495 ]496 ],497 "ps": "[(677, 538)]"498 },499 {500 "step": 33,501 "screenshot": "0084442481728114_33.png",502 "action": "CLICK",503 "info": [504 [505 432,506 534507 ],508 [509 432,510 534511 ]512 ],513 "ps": "[(432, 534)]"514 },515 {516 "step": 34,517 "screenshot": "0084442481728114_34.png",518 "action": "CLICK",519 "info": [520 [521 706,522 679523 ],524 [525 706,526 679527 ]528 ],529 "ps": "[(706, 679)]"530 },531 {532 "step": 35,533 "screenshot": "0084442481728114_35.png",534 "action": "CLICK",535 "info": [536 [537 675,538 528539 ],540 [541 675,542 528543 ]544 ],545 "ps": "[(675, 528)]"546 },547 {548 "step": 36,549 "screenshot": "0084442481728114_36.png",550 "action": "CLICK",551 "info": [552 [553 666,554 543555 ],556 [557 666,558 543559 ]560 ],561 "ps": "[(666, 543)]"562 },563 {564 "step": 37,565 "screenshot": "0084442481728114_37.png",566 "action": "CLICK",567 "info": [568 [569 680,570 718571 ],572 [573 680,574 718575 ]576 ],577 "ps": "[(680, 718)]"578 },579 {580 "step": 38,581 "screenshot": "0084442481728114_38.png",582 "action": "COMPLETE",583 "info": "",584 "ps": "Victor James"585 }586 ]587}