OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0427190330335954",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1840,8 "w": 2208,9 "device_name": "Pixel Fold"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Clock",15 "Opera",16 "Ebay",17 "Facebook"18 ],19 "meta_task": "Plan a movie night by selecting a {} movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",20 "task": "Plan a movie night by selecting a biographical movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",21 "instruction": "Organize a movie night by selecting a biographical film on Opera, adding snacks to your cart on Ebay, and sending out invitations to Victor James using Facebook. Don't forget to set a reminder on the Clock app."22 },23 "step_length": 37,24 "steps": [25 {26 "step": 0,27 "screenshot": "0427190330335954_0.png",28 "action": "CLICK",29 "info": [30 [31 95,32 66533 ],34 [35 95,36 66537 ]38 ],39 "ps": "[(95, 665)]"40 },41 {42 "step": 1,43 "screenshot": "0427190330335954_1.png",44 "action": "CLICK",45 "info": [46 [47 285,48 15349 ],50 [51 285,52 15353 ]54 ],55 "ps": "[(285, 153)]"56 },57 {58 "step": 2,59 "screenshot": "0427190330335954_2.png",60 "action": "TEXT",61 "info": "biographical movies",62 "ps": ""63 },64 {65 "step": 3,66 "screenshot": "0427190330335954_3.png",67 "action": "CLICK",68 "info": [69 [70 900,71 88172 ],73 [74 900,75 88176 ]77 ],78 "ps": "[(900, 881)]"79 },80 {81 "step": 4,82 "screenshot": "0427190330335954_4.png",83 "action": "CLICK",84 "info": [85 [86 548,87 14988 ],89 [90 548,91 14992 ]93 ],94 "ps": "[(548, 149)]"95 },96 {97 "step": 5,98 "screenshot": "0427190330335954_5.png",99 "action": "CLICK",100 "info": [101 [102 921,103 881104 ],105 [106 921,107 881108 ]109 ],110 "ps": "[(921, 881)]"111 },112 {113 "step": 6,114 "screenshot": "0427190330335954_6.png",115 "action": "CLICK",116 "info": [117 [118 341,119 659120 ],121 [122 341,123 659124 ]125 ],126 "ps": "[(341, 659)]"127 },128 {129 "step": 7,130 "screenshot": "0427190330335954_7.png",131 "action": "CLICK",132 "info": [133 [134 101,135 647136 ],137 [138 101,139 647140 ]141 ],142 "ps": "[(101, 647)]"143 },144 {145 "step": 8,146 "screenshot": "0427190330335954_8.png",147 "action": "CLICK",148 "info": "KEY_HOME",149 "ps": ""150 },151 {152 "step": 9,153 "screenshot": "0427190330335954_9.png",154 "action": "CLICK",155 "info": [156 [157 542,158 508159 ],160 [161 542,162 508163 ]164 ],165 "ps": "[(542, 508)]"166 },167 {168 "step": 10,169 "screenshot": "0427190330335954_10.png",170 "action": "CLICK",171 "info": [172 [173 270,174 169175 ],176 [177 270,178 169179 ]180 ],181 "ps": "[(270, 169)]"182 },183 {184 "step": 11,185 "screenshot": "0427190330335954_11.png",186 "action": "TEXT",187 "info": "snacks",188 "ps": ""189 },190 {191 "step": 12,192 "screenshot": "0427190330335954_12.png",193 "action": "CLICK",194 "info": [195 [196 916,197 873198 ],199 [200 916,201 873202 ]203 ],204 "ps": "[(916, 873)]"205 },206 {207 "step": 13,208 "screenshot": "0427190330335954_13.png",209 "action": "CLICK",210 "info": [211 [212 398,213 632214 ],215 [216 398,217 632218 ]219 ],220 "ps": "[(398, 632)]"221 },222 {223 "step": 14,224 "screenshot": "0427190330335954_14.png",225 "action": "SCROLL",226 "info": [227 [228 675,229 870230 ],231 [232 576,233 216234 ]235 ],236 "ps": "[(675, 870), (675, 869), (675, 865), (672, 854), (672, 835), (672, 828), (672, 820), (672, 814), (672, 809), (672, 803), (672, 797), (672, 791), (672, 780), (672, 773), (672, 769), (670, 763), (670, 760), (667, 756), (667, 754), (667, 752), (667, 746), (667, 743), (667, 739), (667, 733), (666, 730), (663, 726), (663, 724), (663, 722), (661, 717), (661, 715), (658, 710), (658, 707), (658, 703), (656, 697), (656, 695), (653, 690), (653, 688), (653, 682), (649, 674), (649, 672), (649, 669), (649, 663), (645, 659), (645, 658), (645, 654), (645, 649), (645, 643), (645, 635), (645, 623), (644, 616), (644, 608), (641, 602), (641, 596), (641, 590), (641, 585), (641, 579), (641, 567), (637, 562), (637, 559), (637, 555), (637, 549), (637, 543), (635, 541), (635, 535), (635, 530), (631, 528), (631, 523), (631, 518), (631, 512), (631, 507), (631, 503), (631, 501), (629, 498), (629, 496), (629, 494), (629, 485), (629, 479), (629, 473), (629, 466), (629, 458), (629, 449), (625, 444), (625, 438), (622, 432), (622, 428), (622, 422), (622, 420), (622, 418), (619, 415), (619, 414), (619, 407), (616, 402), (616, 399), (616, 394), (612, 386), (612, 385), (612, 383), (612, 379), (609, 376), (609, 374), (609, 371), (609, 367), (606, 361), (606, 355), (603, 351), (603, 348), (603, 347), (599, 342), (599, 337), (599, 331), (597, 328), (597, 323), (597, 317), (593, 313), (593, 310), (593, 305), (590, 303), (590, 298), (590, 292), (590, 290), (587, 285), (587, 283), (587, 278), (587, 276), (587, 273), (584, 269), (584, 267), (584, 266), (584, 260), (584, 254), (580, 250), (580, 247), (581, 246), (581, 244), (581, 243), (579, 239), (579, 233), (576, 233), (576, 230), (576, 227), (576, 223), (576, 220), (576, 216)]"237 },238 {239 "step": 15,240 "screenshot": "0427190330335954_15.png",241 "action": "SCROLL",242 "info": [243 [244 573,245 703246 ],247 [248 501,249 276250 ]251 ],252 "ps": "[(573, 703), (573, 699), (569, 693), (569, 689), (569, 682), (569, 673), (568, 670), (567, 663), (565, 656), (561, 647), (560, 639), (555, 626), (552, 617), (552, 606), (552, 599), (552, 588), (552, 580), (552, 575), (552, 564), (549, 560), (549, 558), (549, 555), (549, 552), (549, 548), (546, 545), (546, 541), (546, 538), (546, 535), (542, 530), (542, 526), (540, 523), (540, 519), (540, 516), (536, 512), (536, 509), (536, 503), (536, 498), (536, 496), (536, 494), (533, 486), (533, 484), (533, 478), (529, 472), (529, 471), (529, 466), (529, 461), (529, 455), (529, 449), (529, 444), (529, 438), (529, 432), (529, 431), (529, 424), (527, 421), (527, 415), (523, 409), (523, 402), (519, 397), (519, 391), (516, 385), (516, 379), (513, 374), (513, 370), (510, 364), (507, 358), (507, 355), (507, 349), (507, 344), (507, 338), (507, 333), (507, 327), (507, 321), (507, 313), (507, 307), (507, 305), (507, 303), (507, 297), (503, 292), (503, 290), (503, 285), (503, 283), (503, 278), (501, 278), (501, 276)]"253 },254 {255 "step": 16,256 "screenshot": "0427190330335954_16.png",257 "action": "CLICK",258 "info": [259 [260 519,261 484262 ],263 [264 519,265 484266 ]267 ],268 "ps": "[(519, 484)]"269 },270 {271 "step": 17,272 "screenshot": "0427190330335954_17.png",273 "action": "CLICK",274 "info": "KEY_HOME",275 "ps": ""276 },277 {278 "step": 18,279 "screenshot": "0427190330335954_18.png",280 "action": "CLICK",281 "info": [282 [283 549,284 150285 ],286 [287 549,288 150289 ]290 ],291 "ps": "[(549, 150)]"292 },293 {294 "step": 19,295 "screenshot": "0427190330335954_19.png",296 "action": "CLICK",297 "info": [298 [299 343,300 145301 ],302 [303 343,304 145305 ]306 ],307 "ps": "[(343, 145)]"308 },309 {310 "step": 20,311 "screenshot": "0427190330335954_20.png",312 "action": "CLICK",313 "info": [314 [315 398,316 203317 ],318 [319 398,320 203321 ]322 ],323 "ps": "[(398, 203)]"324 },325 {326 "step": 21,327 "screenshot": "0427190330335954_21.png",328 "action": "CLICK",329 "info": [330 [331 214,332 311333 ],334 [335 214,336 311337 ]338 ],339 "ps": "[(214, 311)]"340 },341 {342 "step": 22,343 "screenshot": "0427190330335954_22.png",344 "action": "SCROLL",345 "info": [346 [347 326,348 894349 ],350 [351 267,352 436353 ]354 ],355 "ps": "[(326, 894), (326, 891), (326, 885), (326, 880), (324, 880), (324, 877), (320, 873), (320, 870), (318, 865), (318, 863), (318, 857), (318, 851), (314, 846), (314, 840), (314, 839), (314, 833), (314, 827), (314, 821), (314, 818), (314, 813), (314, 809), (314, 805), (314, 801), (314, 800), (314, 797), (311, 793), (311, 791), (311, 789), (311, 786), (311, 782), (311, 779), (307, 774), (307, 771), (307, 766), (305, 760), (305, 757), (305, 754), (305, 752), (305, 747), (302, 745), (301, 740), (301, 734), (301, 732), (301, 727), (300, 724), (300, 720), (300, 719), (300, 717), (300, 715), (300, 713), (300, 710), (300, 706), (300, 703), (297, 699), (297, 696), (293, 692), (293, 689), (293, 684), (293, 682), (291, 677), (291, 672), (291, 666), (288, 663), (288, 659), (288, 656), (286, 652), (286, 646), (286, 643), (283, 639), (283, 636), (283, 630), (281, 623), (281, 617), (281, 612), (281, 606), (281, 605), (281, 602), (281, 596), (281, 594), (281, 592), (281, 588), (281, 582), (281, 579), (281, 573), (278, 569), (278, 567), (278, 565), (274, 559), (274, 553), (274, 549), (274, 542), (274, 536), (272, 532), (272, 530), (272, 528), (272, 523), (272, 521), (272, 518), (272, 514), (272, 511), (272, 507), (272, 501), (272, 498), (272, 495), (272, 491), (272, 488), (269, 484), (269, 481), (269, 477), (269, 473), (269, 471), (267, 471), (267, 466), (267, 464), (267, 459), (267, 457), (267, 454), (267, 449), (267, 447), (267, 442), (267, 440), (267, 436)]"356 },357 {358 "step": 23,359 "screenshot": "0427190330335954_23.png",360 "action": "CLICK",361 "info": [362 [363 435,364 586365 ],366 [367 435,368 586369 ]370 ],371 "ps": "[(435, 586)]"372 },373 {374 "step": 24,375 "screenshot": "0427190330335954_24.png",376 "action": "CLICK",377 "info": [378 [379 410,380 522381 ],382 [383 410,384 522385 ]386 ],387 "ps": "[(410, 522)]"388 },389 {390 "step": 25,391 "screenshot": "0427190330335954_25.png",392 "action": "CLICK",393 "info": [394 [395 470,396 918397 ],398 [399 470,400 918401 ]402 ],403 "ps": "[(470, 918)]"404 },405 {406 "step": 26,407 "screenshot": "0427190330335954_26.png",408 "action": "TEXT",409 "info": "Do you want to see a movie with me this Sunday night?",410 "ps": ""411 },412 {413 "step": 27,414 "screenshot": "0427190330335954_27.png",415 "action": "CLICK",416 "info": [417 [418 759,419 505420 ],421 [422 759,423 505424 ]425 ],426 "ps": "[(759, 505)]"427 },428 {429 "step": 28,430 "screenshot": "0427190330335954_28.png",431 "action": "CLICK",432 "info": "KEY_HOME",433 "ps": ""434 },435 {436 "step": 29,437 "screenshot": "0427190330335954_29.png",438 "action": "CLICK",439 "info": [440 [441 444,442 489443 ],444 [445 444,446 489447 ]448 ],449 "ps": "[(444, 489)]"450 },451 {452 "step": 30,453 "screenshot": "0427190330335954_30.png",454 "action": "CLICK",455 "info": [456 [457 760,458 850459 ],460 [461 760,462 850463 ]464 ],465 "ps": "[(760, 850)]"466 },467 {468 "step": 31,469 "screenshot": "0427190330335954_31.png",470 "action": "CLICK",471 "info": [472 [473 371,474 560475 ],476 [477 371,478 560479 ]480 ],481 "ps": "[(371, 560)]"482 },483 {484 "step": 32,485 "screenshot": "0427190330335954_32.png",486 "action": "CLICK",487 "info": [488 [489 554,490 545491 ],492 [493 554,494 545495 ]496 ],497 "ps": "[(554, 545)]"498 },499 {500 "step": 33,501 "screenshot": "0427190330335954_33.png",502 "action": "CLICK",503 "info": [504 [505 615,506 588507 ],508 [509 615,510 588511 ]512 ],513 "ps": "[(615, 588)]"514 },515 {516 "step": 34,517 "screenshot": "0427190330335954_34.png",518 "action": "CLICK",519 "info": [520 [521 804,522 719523 ],524 [525 804,526 719527 ]528 ],529 "ps": "[(804, 719)]"530 },531 {532 "step": 35,533 "screenshot": "0427190330335954_35.png",534 "action": "CLICK",535 "info": [536 [537 630,538 401539 ],540 [541 630,542 401543 ]544 ],545 "ps": "[(630, 401)]"546 },547 {548 "step": 36,549 "screenshot": "0427190330335954_36.png",550 "action": "COMPLETE",551 "info": "",552 "ps": "Victor James"553 }554 ]555}