OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
263.8k
1{2 "episode_id": "1068710401582939",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 3120,8 "w": 1440,9 "device_name": "Pixel 7 Pro"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Ebay",15 "Google Keep",16 "Firefox Browser"17 ],18 "meta_task": "Find a recipe for {}, create a shopping list of main ingredients, and add to the cart.",19 "task": "Find a recipe for Chocolate chip cookies, create a shopping list of main ingredients, and add to the cart.",20 "instruction": "Utilize the Firefox Browser to search for a recipe for Chocolate chip cookies. Once you have located a suitable recipe, use Google Keep to create a shopping list of the main ingredients. Afterward, add these ingredients to your cart on Ebay."21 },22 "step_length": 28,23 "steps": [24 {25 "step": 0,26 "screenshot": "1068710401582939_0.png",27 "action": "CLICK",28 "info": [29 [30 624,31 82632 ],33 [34 624,35 82636 ]37 ],38 "ps": "[(624, 826)]"39 },40 {41 "step": 1,42 "screenshot": "1068710401582939_1.png",43 "action": "CLICK",44 "info": "KEY_HOME",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "1068710401582939_2.png",50 "action": "CLICK",51 "info": [52 [53 644,54 52455 ],56 [57 644,58 52459 ]60 ],61 "ps": "[(644, 524)]"62 },63 {64 "step": 3,65 "screenshot": "1068710401582939_3.png",66 "action": "CLICK",67 "info": [68 [69 834,70 90171 ],72 [73 834,74 90175 ]76 ],77 "ps": "[(834, 901)]"78 },79 {80 "step": 4,81 "screenshot": "1068710401582939_4.png",82 "action": "TEXT",83 "info": "Chocolate chip cookies",84 "ps": ""85 },86 {87 "step": 5,88 "screenshot": "1068710401582939_5.png",89 "action": "TEXT",90 "info": ":150g salted butter, softened 80g light brown muscovado sugar, 80g granulated sugar, 2 tsp vanilla extract, 1 large egg, 225g plain flour, 1/2 tsp bicarbonate of soda, 1/4 tsp salt, 200g plain chocolate chips or chunks",91 "ps": ""92 },93 {94 "step": 6,95 "screenshot": "1068710401582939_6.png",96 "action": "CLICK",97 "info": "KEY_HOME",98 "ps": ""99 },100 {101 "step": 7,102 "screenshot": "1068710401582939_7.png",103 "action": "CLICK",104 "info": [105 [106 146,107 502108 ],109 [110 146,111 502112 ]113 ],114 "ps": "[(146, 502)]"115 },116 {117 "step": 8,118 "screenshot": "1068710401582939_8.png",119 "action": "TEXT",120 "info": "vanilla extract",121 "ps": ""122 },123 {124 "step": 9,125 "screenshot": "1068710401582939_9.png",126 "action": "CLICK",127 "info": [128 [129 904,130 915131 ],132 [133 904,134 915135 ]136 ],137 "ps": "[(904, 915)]"138 },139 {140 "step": 10,141 "screenshot": "1068710401582939_10.png",142 "action": "SCROLL",143 "info": [144 [145 688,146 767147 ],148 [149 678,150 417151 ]152 ],153 "ps": "[(688, 767), (688, 765), (688, 762), (693, 761), (693, 759), (693, 757), (693, 756), (693, 753), (693, 752), (693, 750), (693, 748), (693, 747), (693, 744), (693, 742), (693, 741), (693, 739), (693, 738), (693, 735), (693, 733), (693, 732), (693, 730), (693, 729), (693, 726), (693, 724), (693, 723), (693, 721), (693, 718), (693, 717), (693, 715), (693, 714), (693, 712), (695, 709), (695, 708), (695, 706), (695, 704), (695, 703), (695, 700), (695, 699), (695, 697), (695, 695), (695, 694), (695, 689), (695, 688), (695, 686), (695, 684), (695, 683), (695, 680), (695, 679), (695, 677), (695, 675), (695, 674), (695, 671), (695, 669), (695, 668), (695, 666), (695, 665), (695, 662), (695, 660), (695, 659), (695, 657), (695, 655), (695, 653), (695, 651), (695, 650), (695, 648), (695, 646), (695, 644), (695, 642), (695, 640), (695, 639), (700, 639), (700, 637), (700, 635), (700, 633), (700, 631), (700, 630), (700, 628), (700, 626), (700, 624), (700, 622), (700, 621), (700, 619), (700, 616), (700, 615), (700, 613), (700, 612), (700, 610), (700, 607), (700, 606), (700, 604), (700, 602), (700, 601), (700, 599), (700, 597), (700, 595), (700, 593), (700, 592), (700, 590), (700, 587), (700, 586), (700, 584), (700, 583), (700, 581), (700, 578), (700, 577), (700, 575), (700, 574), (700, 572), (700, 569), (700, 568), (700, 566), (700, 564), (700, 563), (700, 560), (700, 559), (700, 557), (700, 555), (700, 554), (700, 551), (700, 549), (700, 548), (700, 546), (700, 545), (700, 542), (700, 540), (700, 539), (700, 537), (700, 534), (700, 533), (700, 531), (700, 530), (700, 528), (700, 525), (700, 524), (700, 522), (700, 520), (700, 519), (700, 516), (700, 516), (700, 513), (700, 511), (700, 510), (700, 508), (700, 507), (700, 504), (697, 504), (697, 502), (697, 501), (697, 499), (697, 496), (697, 495), (697, 493), (697, 492), (697, 490), (697, 487), (697, 486), (697, 484), (697, 482), (697, 481), (697, 478), (697, 477), (697, 475), (697, 473), (697, 472), (695, 469), (695, 467), (695, 466), (695, 464), (695, 463), (695, 460), (695, 458), (695, 457), (695, 455), (695, 453), (695, 451), (695, 449), (695, 448), (695, 446), (690, 444), (690, 442), (690, 440), (690, 439), (690, 437), (690, 435), (690, 433), (690, 431), (686, 431), (686, 429), (686, 428), (686, 426), (686, 424), (686, 422), (683, 422), (683, 420), (683, 419), (678, 419), (678, 417)]"154 },155 {156 "step": 11,157 "screenshot": "1068710401582939_11.png",158 "action": "CLICK",159 "info": [160 [161 751,162 677163 ],164 [165 751,166 677167 ]168 ],169 "ps": "[(751, 677)]"170 },171 {172 "step": 12,173 "screenshot": "1068710401582939_12.png",174 "action": "SCROLL",175 "info": [176 [177 608,178 845179 ],180 [181 634,182 367183 ]184 ],185 "ps": "[(608, 845), (608, 839), (608, 837), (608, 835), (608, 834), (608, 832), (610, 830), (610, 828), (610, 826), (610, 824), (610, 823), (610, 821), (610, 819), (610, 817), (610, 815), (610, 814), (610, 809), (610, 808), (610, 806), (610, 804), (610, 803), (610, 800), (610, 799), (610, 797), (610, 795), (610, 794), (610, 791), (610, 789), (610, 788), (610, 786), (610, 785), (610, 782), (610, 780), (610, 779), (610, 777), (610, 776), (610, 774), (610, 771), (610, 770), (610, 768), (610, 765), (608, 762), (608, 760), (608, 759), (608, 757), (608, 754), (608, 753), (608, 751), (608, 750), (608, 748), (608, 744), (608, 740), (608, 735), (608, 734), (608, 732), (608, 730), (608, 726), (608, 724), (608, 723), (608, 721), (608, 718), (608, 717), (608, 715), (608, 712), (608, 707), (608, 706), (608, 704), (608, 702), (608, 700), (608, 698), (608, 696), (608, 695), (608, 693), (608, 691), (608, 689), (608, 687), (608, 686), (608, 684), (608, 682), (608, 680), (608, 678), (608, 677), (608, 675), (608, 673), (608, 671), (608, 667), (608, 666), (608, 664), (608, 661), (608, 660), (608, 658), (608, 657), (608, 655), (608, 652), (608, 651), (608, 649), (608, 647), (608, 646), (608, 643), (608, 642), (608, 640), (608, 638), (608, 637), (608, 634), (608, 632), (608, 631), (608, 629), (608, 628), (608, 625), (608, 623), (608, 622), (608, 620), (608, 617), (608, 616), (608, 614), (608, 613), (608, 611), (608, 608), (608, 607), (608, 605), (608, 604), (608, 602), (608, 599), (608, 599), (608, 596), (608, 594), (608, 593), (608, 591), (608, 590), (608, 587), (608, 585), (610, 584), (610, 582), (610, 579), (610, 578), (610, 576), (610, 575), (610, 573), (610, 570), (610, 569), (610, 567), (610, 565), (610, 564), (610, 561), (610, 560), (615, 558), (615, 556), (615, 555), (615, 552), (615, 550), (615, 549), (615, 547), (615, 546), (615, 543), (615, 541), (618, 540), (618, 538), (618, 537), (618, 534), (618, 532), (618, 531), (618, 529), (618, 527), (618, 525), (618, 523), (618, 522), (618, 520), (618, 518), (618, 516), (622, 514), (622, 512), (622, 511), (622, 509), (622, 508), (622, 505), (622, 503), (622, 502), (622, 500), (622, 498), (622, 496), (622, 494), (622, 493), (622, 491), (622, 489), (622, 487), (622, 485), (622, 484), (622, 482), (622, 480), (622, 478), (622, 476), (622, 474), (624, 473), (624, 471), (624, 469), (624, 467), (624, 465), (624, 464), (624, 462), (624, 459), (624, 458), (624, 456), (624, 455), (624, 453), (629, 453), (629, 450), (629, 449), (629, 447), (629, 445), (629, 444), (629, 441), (629, 440), (629, 438), (629, 436), (629, 435), (629, 432), (629, 430), (629, 429), (629, 427), (629, 426), (629, 424), (629, 421), (629, 420), (629, 418), (629, 417), (629, 415), (629, 412), (629, 411), (629, 409), (629, 407), (629, 406), (629, 403), (629, 402), (629, 400), (629, 398), (634, 397), (634, 394), (634, 392), (634, 391), (634, 389), (634, 388), (634, 385), (634, 383), (634, 382), (634, 380), (634, 377), (634, 376), (634, 374), (634, 373), (634, 371), (634, 368), (634, 367)]"186 },187 {188 "step": 13,189 "screenshot": "1068710401582939_13.png",190 "action": "CLICK",191 "info": [192 [193 297,194 709195 ],196 [197 297,198 709199 ]200 ],201 "ps": "[(297, 709)]"202 },203 {204 "step": 14,205 "screenshot": "1068710401582939_14.png",206 "action": "CLICK",207 "info": [208 [209 447,210 407211 ],212 [213 447,214 407215 ]216 ],217 "ps": "[(447, 407)]"218 },219 {220 "step": 15,221 "screenshot": "1068710401582939_15.png",222 "action": "CLICK",223 "info": [224 [225 574,226 72227 ],228 [229 574,230 72231 ]232 ],233 "ps": "[(574, 72)]"234 },235 {236 "step": 16,237 "screenshot": "1068710401582939_16.png",238 "action": "CLICK",239 "info": [240 [241 793,242 74243 ],244 [245 793,246 74247 ]248 ],249 "ps": "[(793, 74)]"250 },251 {252 "step": 17,253 "screenshot": "1068710401582939_17.png",254 "action": "TEXT",255 "info": "egg",256 "ps": ""257 },258 {259 "step": 18,260 "screenshot": "1068710401582939_18.png",261 "action": "CLICK",262 "info": [263 [264 895,265 890266 ],267 [268 895,269 890270 ]271 ],272 "ps": "[(895, 890)]"273 },274 {275 "step": 19,276 "screenshot": "1068710401582939_19.png",277 "action": "SCROLL",278 "info": [279 [280 600,281 765282 ],283 [284 644,285 352286 ]287 ],288 "ps": "[(600, 765), (603, 762), (603, 760), (603, 758), (603, 757), (603, 754), (608, 754), (608, 752), (608, 751), (608, 749), (608, 748), (613, 748), (613, 745), (613, 743), (613, 742), (613, 740), (613, 738), (613, 736), (615, 736), (615, 734), (615, 733), (615, 731), (615, 729), (615, 727), (620, 727), (620, 725), (620, 724), (620, 722), (620, 720), (620, 718), (620, 716), (620, 714), (620, 713), (620, 711), (620, 709), (620, 707), (620, 705), (620, 704), (620, 702), (622, 699), (622, 698), (622, 696), (622, 695), (622, 693), (622, 691), (622, 689), (622, 687), (622, 685), (622, 682), (622, 679), (622, 678), (622, 676), (622, 675), (622, 673), (622, 670), (622, 669), (622, 667), (622, 665), (622, 661), (622, 659), (622, 658), (622, 656), (622, 653), (622, 652), (622, 650), (622, 649), (622, 644), (622, 643), (622, 641), (622, 639), (622, 635), (622, 633), (622, 632), (622, 630), (622, 626), (622, 624), (618, 622), (618, 619), (618, 616), (618, 615), (618, 613), (618, 611), (618, 610), (618, 607), (618, 606), (618, 604), (618, 602), (618, 601), (618, 599), (618, 596), (618, 595), (618, 593), (615, 592), (615, 590), (615, 587), (615, 586), (615, 584), (615, 582), (615, 581), (615, 578), (615, 577), (615, 575), (615, 573), (615, 572), (615, 569), (615, 567), (615, 566), (615, 564), (615, 563), (615, 560), (615, 558), (615, 557), (615, 555), (615, 554), (615, 551), (615, 549), (615, 548), (615, 546), (615, 543), (615, 542), (615, 540), (615, 539), (615, 537), (615, 534), (615, 533), (615, 531), (615, 529), (615, 528), (615, 525), (615, 524), (615, 522), (615, 520), (615, 519), (615, 516), (615, 514), (615, 513), (615, 511), (615, 510), (615, 508), (615, 505), (615, 504), (620, 502), (620, 500), (620, 499), (620, 496), (620, 495), (620, 493), (620, 491), (620, 490), (620, 487), (620, 486), (620, 484), (622, 482), (622, 481), (622, 478), (622, 476), (622, 475), (622, 473), (622, 472), (627, 472), (627, 469), (627, 467), (627, 466), (627, 464), (627, 462), (627, 460), (627, 458), (627, 457), (627, 455), (632, 453), (632, 451), (632, 449), (632, 447), (632, 446), (632, 444), (634, 442), (634, 440), (634, 438), (634, 437), (634, 435), (634, 433), (634, 431), (634, 429), (634, 428), (634, 426), (634, 424), (634, 422), (639, 422), (639, 420), (639, 419), (639, 417), (639, 415), (639, 413), (639, 411), (639, 409), (639, 408), (639, 406), (639, 404), (639, 402), (639, 400), (639, 399), (639, 397), (644, 397), (644, 394), (644, 393), (644, 391), (644, 390), (644, 388), (644, 385), (644, 384), (644, 382), (644, 380), (644, 379), (644, 376), (644, 375), (644, 373), (644, 371), (644, 370), (644, 367), (644, 366), (644, 364), (644, 362), (644, 361), (644, 358), (644, 356), (644, 355), (644, 353), (644, 352)]"289 },290 {291 "step": 20,292 "screenshot": "1068710401582939_20.png",293 "action": "CLICK",294 "info": [295 [296 695,297 79298 ],299 [300 695,301 79302 ]303 ],304 "ps": "[(695, 79)]"305 },306 {307 "step": 21,308 "screenshot": "1068710401582939_21.png",309 "action": "CLICK",310 "info": [311 [312 783,313 79314 ],315 [316 783,317 79318 ]319 ],320 "ps": "[(783, 79)]"321 },322 {323 "step": 22,324 "screenshot": "1068710401582939_22.png",325 "action": "TEXT",326 "info": "plain chocolate chips",327 "ps": ""328 },329 {330 "step": 23,331 "screenshot": "1068710401582939_23.png",332 "action": "CLICK",333 "info": [334 [335 911,336 898337 ],338 [339 911,340 898341 ]342 ],343 "ps": "[(911, 898)]"344 },345 {346 "step": 24,347 "screenshot": "1068710401582939_24.png",348 "action": "CLICK",349 "info": [350 [351 686,352 749353 ],354 [355 686,356 749357 ]358 ],359 "ps": "[(686, 749)]"360 },361 {362 "step": 25,363 "screenshot": "1068710401582939_25.png",364 "action": "SCROLL",365 "info": [366 [367 263,368 828369 ],370 [371 326,372 383373 ]374 ],375 "ps": "[(263, 828), (263, 827), (263, 826), (263, 824), (263, 822), (263, 821), (268, 818), (268, 817), (268, 815), (270, 815), (270, 813), (270, 812), (270, 809), (270, 807), (270, 806), (274, 804), (274, 803), (274, 800), (274, 798), (274, 797), (274, 795), (274, 792), (274, 791), (274, 789), (279, 788), (279, 786), (279, 783), (279, 782), (279, 780), (279, 778), (279, 777), (279, 772), (279, 771), (279, 769), (279, 768), (279, 766), (279, 763), (279, 762), (279, 760), (279, 757), (279, 754), (279, 752), (279, 751), (279, 749), (279, 748), (279, 743), (279, 741), (279, 740), (279, 738), (279, 736), (279, 734), (279, 732), (279, 731), (279, 729), (279, 726), (279, 725), (279, 723), (279, 722), (279, 720), (279, 717), (279, 716), (279, 712), (279, 708), (279, 704), (279, 699), (279, 698), (279, 696), (279, 695), (279, 691), (279, 686), (279, 685), (279, 683), (279, 682), (279, 679), (279, 677), (279, 676), (279, 671), (279, 670), (279, 668), (279, 666), (279, 662), (279, 660), (279, 659), (279, 655), (279, 653), (279, 651), (279, 649), (279, 648), (279, 643), (279, 642), (279, 640), (279, 638), (279, 634), (279, 632), (279, 631), (279, 629), (279, 628), (279, 623), (279, 621), (279, 620), (279, 617), (279, 616), (279, 612), (279, 611), (279, 608), (279, 606), (279, 605), (279, 601), (279, 599), (279, 595), (279, 592), (279, 587), (279, 585), (279, 584), (279, 582), (279, 579), (279, 578), (279, 576), (279, 573), (279, 570), (279, 568), (279, 567), (279, 565), (279, 564), (279, 559), (279, 555), (279, 551), (279, 549), (279, 548), (282, 546), (282, 543), (282, 542), (282, 540), (282, 538), (282, 537), (282, 534), (282, 533), (282, 531), (282, 529), (282, 528), (282, 525), (287, 524), (287, 522), (287, 520), (287, 519), (287, 514), (287, 513), (287, 511), (287, 509), (287, 508), (287, 505), (287, 503), (287, 502), (292, 500), (292, 499), (292, 496), (292, 494), (292, 493), (292, 491), (292, 490), (292, 487), (292, 485), (292, 484), (292, 482), (292, 480), (292, 478), (292, 476), (294, 475), (294, 473), (294, 471), (294, 469), (294, 467), (294, 465), (294, 464), (294, 462), (294, 460), (294, 458), (294, 456), (299, 455), (299, 453), (299, 450), (299, 449), (299, 447), (299, 446), (299, 444), (302, 444), (302, 441), (302, 440), (302, 438), (302, 436), (302, 435), (302, 432), (302, 431), (302, 429), (306, 429), (306, 427), (306, 426), (306, 424), (306, 422), (311, 420), (311, 418), (311, 417), (311, 415), (313, 403), (313, 402), (318, 400), (318, 398), (323, 389), (323, 388), (323, 385), (326, 385), (326, 383)]"376 },377 {378 "step": 26,379 "screenshot": "1068710401582939_26.png",380 "action": "CLICK",381 "info": [382 [383 219,384 736385 ],386 [387 219,388 736389 ]390 ],391 "ps": "[(219, 736)]"392 },393 {394 "step": 27,395 "screenshot": "1068710401582939_27.png",396 "action": "COMPLETE",397 "info": "",398 "ps": ""399 }400 ]401}