OpenGVLab/GUI-Odyssey

1{2    "episode_id": "1084880108783314",3    "device_info": {4        "product": "sdk_gphone64_x86_64",5        "release_version": "14",6        "sdk_version": "34",7        "h": 1280,8        "w": 720,9        "device_name": "Small Phone"10    },11    "task_info": {12        "category": "Multi_Apps",13        "app": [14            "Amazon",15            "TickTick",16            "Firefox Browser"17        ],18        "meta_task": "Find a recipe for {}, create a shopping list of main ingredients, and add to the cart.",19        "task": "Find a recipe for Chocolate chip cookies, create a shopping list of main ingredients, and add to the cart.",20        "instruction": "Start by using the Firefox Browser to find a recipe for chocolate chip cookies. Next, create a shopping list of the main ingredients using TickTick. Finally, add these items to your cart on Amazon."21    },22    "step_length": 23,23    "steps": [24        {25            "step": 0,26            "screenshot": "1084880108783314_0.png",27            "action": "CLICK",28            "info": [29                [30                    624,31                    74332                ],33                [34                    624,35                    74336                ]37            ],38            "ps": "[(624, 743)]"39        },40        {41            "step": 1,42            "screenshot": "1084880108783314_1.png",43            "action": "CLICK",44            "info": "KEY_HOME",45            "ps": ""46        },47        {48            "step": 2,49            "screenshot": "1084880108783314_2.png",50            "action": "CLICK",51            "info": [52                [53                    338,54                    32655                ],56                [57                    338,58                    32659                ]60            ],61            "ps": "[(338, 326)]"62        },63        {64            "step": 3,65            "screenshot": "1084880108783314_3.png",66            "action": "CLICK",67            "info": [68                [69                    874,70                    80671                ],72                [73                    874,74                    80675                ]76            ],77            "ps": "[(874, 806)]"78        },79        {80            "step": 4,81            "screenshot": "1084880108783314_4.png",82            "action": "TEXT",83            "info": "Chocolate chip cookies: 150g salted butter, softened 80g light brown muscovado sugar, 80g granulated sugar, 2 tsp vanilla extract, 1 large egg, 225g plain flour, 1/2 tsp bicarbonate of soda, 1/4 tsp salt, 200g plain chocolate chips or chunks",84            "ps": ""85        },86        {87            "step": 5,88            "screenshot": "1084880108783314_5.png",89            "action": "CLICK",90            "info": [91                [92                    888,93                    48494                ],95                [96                    888,97                    48498                ]99            ],100            "ps": "[(888, 484)]"101        },102        {103            "step": 6,104            "screenshot": "1084880108783314_6.png",105            "action": "CLICK",106            "info": "KEY_HOME",107            "ps": ""108        },109        {110            "step": 7,111            "screenshot": "1084880108783314_7.png",112            "action": "CLICK",113            "info": [114                [115                    624,116                    457117                ],118                [119                    624,120                    457121                ]122            ],123            "ps": "[(624, 457)]"124        },125        {126            "step": 8,127            "screenshot": "1084880108783314_8.png",128            "action": "CLICK",129            "info": [130                [131                    233,132                    76133                ],134                [135                    233,136                    76137                ]138            ],139            "ps": "[(233, 76)]"140        },141        {142            "step": 9,143            "screenshot": "1084880108783314_9.png",144            "action": "TEXT",145            "info": "vanilla extract",146            "ps": ""147        },148        {149            "step": 10,150            "screenshot": "1084880108783314_10.png",151            "action": "CLICK",152            "info": [153                [154                    927,155                    867156                ],157                [158                    927,159                    867160                ]161            ],162            "ps": "[(927, 867)]"163        },164        {165            "step": 11,166            "screenshot": "1084880108783314_11.png",167            "action": "CLICK",168            "info": [169                [170                    669,171                    782172                ],173                [174                    669,175                    782176                ]177            ],178            "ps": "[(669, 782)]"179        },180        {181            "step": 12,182            "screenshot": "1084880108783314_12.png",183            "action": "SCROLL",184            "info": [185                [186                    347,187                    776188                ],189                [190                    563,191                    10192                ]193            ],194            "ps": "[(347, 776), (347, 774), (347, 771), (347, 770), (347, 767), (352, 765), (352, 762), (352, 759), (355, 759), (355, 757), (355, 754), (361, 753), (361, 749), (361, 746), (366, 745), (366, 742), (366, 739), (369, 737), (369, 734), (369, 732), (374, 729), (374, 726), (374, 724), (374, 721), (380, 718), (380, 717), (380, 714), (380, 712), (383, 709), (383, 706), (383, 704), (383, 701), (383, 698), (388, 696), (388, 693), (388, 692), (388, 689), (391, 685), (391, 684), (391, 681), (397, 678), (397, 676), (397, 673), (397, 671), (397, 668), (402, 665), (402, 664), (402, 660), (402, 659), (405, 656), (405, 653), (405, 651), (405, 645), (411, 639), (411, 634), (411, 631), (416, 631), (416, 626), (416, 620), (416, 617), (416, 615), (419, 612), (419, 609), (419, 607), (419, 604), (419, 603), (419, 599), (419, 596), (424, 595), (424, 592), (424, 589), (424, 587), (424, 584), (430, 582), (430, 579), (430, 576), (430, 574), (430, 571), (430, 570), (430, 567), (430, 564), (430, 562), (430, 559), (430, 556), (433, 556), (433, 554), (433, 551), (433, 549), (433, 546), (433, 543), (438, 542), (438, 539), (438, 535), (438, 534), (438, 531), (438, 529), (438, 526), (438, 523), (438, 521), (438, 518), (438, 515), (444, 515), (444, 514), (444, 510), (444, 509), (444, 506), (444, 503), (444, 501), (444, 498), (444, 496), (447, 496), (447, 493), (447, 490), (447, 489), (447, 485), (447, 482), (447, 481), (447, 478), (447, 476), (447, 473), (447, 470), (452, 468), (452, 465), (452, 462), (452, 460), (452, 457), (452, 456), (452, 453), (455, 449), (455, 448), (455, 445), (455, 442), (455, 440), (455, 437), (455, 435), (455, 432), (455, 429), (455, 428), (455, 424), (461, 424), (461, 421), (461, 420), (461, 417), (461, 415), (461, 412), (461, 409), (461, 407), (466, 404), (466, 403), (466, 399), (466, 396), (466, 395), (466, 392), (466, 389), (466, 387), (466, 384), (466, 382), (466, 379), (466, 376), (469, 374), (469, 371), (469, 368), (469, 367), (469, 364), (469, 362), (474, 359), (474, 356), (474, 354), (474, 351), (474, 348), (474, 346), (474, 343), (474, 342), (474, 339), (474, 335), (480, 334), (480, 331), (480, 328), (480, 326), (480, 323), (480, 321), (483, 318), (483, 315), (483, 314), (483, 310), (483, 309), (483, 306), (483, 303), (483, 301), (488, 298), (488, 295), (488, 293), (488, 290), (488, 289), (488, 285), (491, 282), (491, 281), (491, 278), (491, 274), (491, 273), (491, 270), (491, 268), (491, 265), (491, 262), (497, 260), (497, 257), (497, 254), (497, 253), (497, 249), (497, 248), (502, 245), (502, 242), (502, 240), (502, 237), (502, 235), (502, 229), (502, 223), (502, 221), (505, 218), (505, 217), (505, 214), (505, 210), (511, 209), (511, 206), (511, 201), (511, 195), (511, 192), (511, 190), (511, 187), (511, 184), (516, 182), (516, 179), (516, 178), (516, 171), (516, 170), (519, 167), (519, 164), (519, 162), (519, 159), (519, 156), (519, 151), (519, 145), (524, 145), (524, 140), (524, 137), (524, 135), (524, 132), (527, 129), (527, 128), (527, 124), (527, 123), (527, 120), (527, 117), (527, 115), (533, 112), (533, 109), (533, 107), (533, 104), (533, 103), (533, 99), (533, 96), (538, 95), (538, 92), (538, 89), (538, 87), (538, 84), (541, 82), (541, 79), (541, 76), (541, 74), (541, 71), (541, 70), (547, 67), (547, 64), (547, 62), (547, 56), (552, 54), (552, 51), (552, 48), (552, 46), (552, 43), (555, 40), (555, 39), (555, 35), (555, 34), (555, 31), (555, 28), (561, 28), (561, 26), (561, 23), (561, 21), (561, 18), (561, 15), (561, 14), (563, 14), (563, 10)]"195        },196        {197            "step": 13,198            "screenshot": "1084880108783314_13.png",199            "action": "SCROLL",200            "info": [201                [202                    322,203                    812204                ],205                [206                    447,207                    221208                ]209            ],210            "ps": "[(322, 812), (322, 809), (324, 809), (324, 807), (324, 804), (324, 801), (324, 799), (324, 796), (324, 795), (324, 792), (324, 789), (324, 787), (324, 784), (330, 784), (330, 781), (330, 779), (330, 776), (330, 774), (330, 771), (330, 768), (330, 767), (330, 764), (336, 764), (336, 760), (336, 759), (336, 756), (336, 754), (336, 751), (336, 748), (336, 746), (338, 743), (338, 742), (338, 739), (338, 735), (338, 734), (338, 731), (344, 731), (344, 728), (344, 726), (344, 723), (344, 721), (344, 718), (344, 715), (344, 714), (344, 710), (344, 707), (344, 706), (344, 703), (349, 703), (349, 701), (349, 698), (349, 695), (349, 693), (349, 690), (349, 687), (349, 685), (349, 682), (352, 681), (352, 678), (352, 674), (352, 673), (352, 670), (352, 668), (352, 665), (352, 662), (352, 660), (352, 657), (352, 654), (358, 653), (358, 649), (358, 648), (358, 645), (358, 642), (358, 640), (358, 637), (358, 634), (361, 632), (361, 629), (361, 628), (361, 624), (361, 621), (361, 620), (361, 617), (361, 614), (361, 612), (361, 609), (366, 607), (366, 604), (366, 601), (366, 599), (366, 596), (366, 593), (366, 592), (372, 589), (372, 587), (372, 584), (372, 581), (372, 579), (372, 576), (372, 574), (372, 571), (372, 568), (372, 567), (374, 564), (374, 560), (374, 559), (374, 556), (374, 554), (374, 551), (380, 548), (380, 546), (380, 543), (380, 540), (380, 539), (380, 535), (380, 534), (380, 531), (380, 528), (380, 526), (380, 523), (386, 520), (386, 518), (386, 515), (386, 514), (386, 510), (386, 507), (386, 506), (386, 503), (388, 503), (388, 499), (388, 498), (388, 495), (388, 493), (388, 490), (388, 487), (388, 485), (388, 482), (388, 481), (394, 478), (394, 474), (394, 473), (394, 467), (394, 465), (394, 462), (394, 459), (397, 457), (397, 454), (397, 451), (397, 446), (397, 443), (397, 442), (397, 439), (397, 437), (397, 434), (402, 431), (402, 429), (402, 426), (402, 423), (402, 421), (402, 418), (402, 417), (408, 414), (408, 410), (408, 409), (408, 406), (408, 403), (408, 401), (408, 398), (408, 396), (408, 393), (411, 390), (411, 389), (411, 385), (411, 384), (411, 381), (411, 378), (411, 376), (416, 373), (416, 370), (416, 368), (416, 365), (416, 364), (416, 360), (416, 357), (416, 356), (416, 353), (419, 353), (419, 349), (419, 348), (419, 345), (419, 343), (419, 340), (419, 337), (424, 337), (424, 335), (424, 332), (424, 329), (424, 328), (424, 324), (424, 323), (424, 320), (424, 317), (424, 315), (430, 312), (430, 309), (430, 307), (430, 304), (430, 303), (430, 299), (433, 299), (433, 296), (433, 295), (433, 292), (433, 290), (433, 287), (433, 284), (433, 282), (433, 279), (433, 276), (433, 274), (438, 274), (438, 271), (438, 270), (438, 267), (438, 264), (438, 262), (444, 262), (444, 259), (444, 256), (444, 254), (444, 251), (444, 249), (444, 246), (444, 243), (444, 242), (444, 239), (444, 235), (444, 234), (447, 234), (447, 231), (447, 229), (447, 226), (447, 223), (447, 221)]"211        },212        {213            "step": 14,214            "screenshot": "1084880108783314_14.png",215            "action": "CLICK",216            "info": [217                [218                    349,219                    560220                ],221                [222                    349,223                    560224                ]225            ],226            "ps": "[(349, 560)]"227        },228        {229            "step": 15,230            "screenshot": "1084880108783314_15.png",231            "action": "CLICK",232            "info": [233                [234                    611,235                    96236                ],237                [238                    611,239                    96240                ]241            ],242            "ps": "[(611, 96)]"243        },244        {245            "step": 16,246            "screenshot": "1084880108783314_16.png",247            "action": "CLICK",248            "info": [249                [250                    905,251                    82252                ],253                [254                    905,255                    82256                ]257            ],258            "ps": "[(905, 82)]"259        },260        {261            "step": 17,262            "screenshot": "1084880108783314_17.png",263            "action": "TEXT",264            "info": "plain chocolate chips",265            "ps": ""266        },267        {268            "step": 18,269            "screenshot": "1084880108783314_18.png",270            "action": "CLICK",271            "info": [272                [273                    899,274                    890275                ],276                [277                    899,278                    890279                ]280            ],281            "ps": "[(899, 890)]"282        },283        {284            "step": 19,285            "screenshot": "1084880108783314_19.png",286            "action": "SCROLL",287            "info": [288                [289                    408,290                    818291                ],292                [293                    547,294                    0295                ]296            ],297            "ps": "[(408, 818), (422, 793), (422, 792), (427, 789), (427, 785), (427, 784), (427, 781), (427, 778), (430, 778), (430, 776), (430, 773), (430, 771), (430, 768), (430, 765), (430, 764), (430, 760), (436, 760), (436, 757), (436, 756), (436, 753), (436, 751), (436, 748), (436, 745), (436, 743), (436, 740), (436, 737), (436, 735), (441, 735), (441, 732), (441, 731), (441, 728), (441, 724), (441, 723), (441, 720), (441, 718), (441, 715), (441, 712), (441, 710), (441, 707), (441, 704), (441, 703), (441, 699), (441, 698), (441, 695), (441, 692), (441, 690), (441, 687), (441, 684), (441, 682), (441, 679), (441, 678), (441, 674), (441, 671), (441, 670), (441, 667), (441, 664), (441, 662), (441, 659), (441, 657), (441, 654), (441, 651), (441, 649), (441, 646), (441, 643), (441, 642), (441, 639), (441, 637), (441, 634), (441, 631), (441, 629), (441, 626), (441, 624), (441, 621), (441, 618), (441, 617), (441, 614), (441, 610), (441, 609), (441, 606), (441, 604), (441, 601), (441, 598), (441, 596), (441, 593), (441, 590), (441, 589), (441, 585), (441, 584), (441, 581), (441, 578), (441, 576), (441, 573), (441, 570), (441, 568), (441, 565), (441, 564), (441, 560), (441, 557), (441, 556), (441, 553), (441, 549), (441, 548), (441, 545), (441, 543), (441, 540), (441, 537), (441, 535), (441, 532), (441, 531), (441, 528), (441, 524), (441, 523), (441, 520), (441, 517), (441, 515), (441, 512), (441, 510), (441, 507), (441, 504), (441, 503), (441, 499), (441, 496), (438, 495), (438, 492), (438, 490), (438, 487), (438, 484), (438, 482), (438, 479), (438, 476), (438, 474), (438, 471), (438, 470), (438, 467), (438, 464), (438, 462), (438, 459), (438, 457), (438, 454), (438, 451), (438, 449), (438, 446), (438, 443), (438, 442), (438, 439), (438, 437), (438, 434), (438, 431), (438, 429), (441, 429), (441, 426), (441, 423), (441, 421), (441, 418), (441, 417), (441, 414), (441, 410), (441, 409), (441, 406), (441, 403), (441, 401), (444, 398), (444, 396), (444, 393), (444, 390), (444, 389), (444, 385), (444, 382), (444, 381), (444, 378), (444, 376), (449, 373), (449, 370), (449, 368), (449, 365), (449, 364), (449, 360), (449, 357), (449, 356), (452, 356), (452, 353), (452, 349), (452, 348), (452, 345), (452, 343), (452, 340), (452, 337), (452, 335), (452, 332), (458, 329), (458, 328), (458, 324), (458, 323), (458, 320), (458, 317), (463, 317), (463, 315), (463, 312), (463, 309), (463, 307), (463, 304), (463, 303), (463, 299), (463, 296), (463, 295), (466, 292), (466, 290), (466, 287), (466, 284), (466, 282), (466, 279), (472, 279), (472, 276), (472, 274), (472, 271), (472, 270), (472, 267), (472, 264), (472, 262), (472, 259), (474, 256), (474, 254), (474, 251), (474, 249), (474, 246), (474, 243), (480, 242), (480, 239), (480, 235), (480, 234), (480, 231), (480, 229), (480, 226), (486, 223), (486, 221), (486, 218), (486, 215), (486, 214), (486, 210), (486, 209), (486, 206), (486, 203), (488, 203), (488, 201), (488, 198), (488, 196), (488, 193), (488, 190), (494, 190), (494, 189), (494, 185), (494, 182), (494, 181), (494, 178), (494, 176), (494, 173), (494, 170), (499, 168), (499, 165), (499, 162), (499, 160), (499, 157), (502, 157), (502, 156), (502, 153), (502, 149), (502, 148), (502, 145), (502, 142), (502, 140), (508, 140), (508, 137), (508, 135), (508, 132), (508, 129), (508, 128), (508, 124), (511, 124), (511, 121), (511, 120), (511, 117), (511, 115), (516, 115), (516, 112), (516, 109), (516, 107), (516, 104), (516, 103), (516, 99), (522, 99), (522, 96), (522, 95), (522, 92), (522, 89), (524, 87), (524, 84), (524, 82), (524, 79), (530, 79), (530, 76), (530, 74), (530, 71), (530, 68), (530, 67), (530, 64), (530, 62), (530, 59), (530, 56), (536, 56), (536, 54), (536, 51), (536, 48), (536, 46), (536, 43), (536, 42), (536, 39), (538, 39), (538, 35), (538, 34), (538, 31), (538, 29), (538, 26), (544, 26), (544, 23), (544, 21), (544, 18), (544, 15), (544, 14), (544, 10), (544, 9), (547, 9), (547, 6), (547, 3), (547, 1), (547, 0)]"298        },299        {300            "step": 20,301            "screenshot": "1084880108783314_20.png",302            "action": "SCROLL",303            "info": [304                [305                    411,306                    754307                ],308                [309                    549,310                    249311                ]312            ],313            "ps": "[(411, 754), (413, 753), (413, 749), (419, 749), (419, 746), (424, 746), (424, 745), (427, 745), (427, 742), (427, 739), (433, 739), (433, 737), (433, 734), (438, 732), (438, 729), (441, 726), (441, 724), (447, 721), (447, 720), (449, 717), (449, 714), (449, 712), (449, 709), (455, 709), (455, 706), (455, 704), (461, 704), (461, 701), (461, 699), (461, 696), (461, 693), (463, 693), (463, 692), (469, 689), (469, 685), (469, 684), (469, 681), (469, 679), (474, 676), (474, 673), (477, 671), (477, 668), (477, 665), (477, 664), (483, 660), (483, 659), (483, 656), (483, 653), (483, 651), (483, 648), (486, 648), (486, 646), (486, 643), (486, 640), (486, 639), (491, 639), (491, 635), (491, 632), (491, 631), (491, 628), (491, 626), (491, 623), (497, 620), (497, 618), (497, 615), (497, 612), (497, 610), (497, 607), (499, 607), (499, 606), (499, 603), (499, 599), (499, 598), (499, 595), (499, 592), (499, 587), (499, 584), (499, 582), (499, 579), (499, 578), (499, 574), (505, 571), (505, 570), (505, 567), (505, 564), (505, 562), (508, 559), (508, 557), (508, 554), (508, 548), (508, 546), (508, 543), (508, 542), (508, 539), (508, 535), (513, 534), (513, 531), (513, 529), (513, 526), (513, 523), (519, 521), (519, 518), (519, 515), (519, 514), (519, 507), (519, 506), (519, 503), (519, 499), (519, 498), (522, 495), (522, 493), (522, 490), (522, 487), (522, 485), (522, 482), (522, 481), (522, 478), (522, 474), (522, 473), (527, 470), (527, 467), (527, 465), (527, 462), (527, 460), (527, 457), (527, 454), (527, 453), (527, 449), (527, 446), (527, 445), (527, 442), (527, 440), (527, 437), (533, 434), (533, 432), (533, 429), (533, 426), (533, 424), (533, 421), (533, 420), (533, 414), (533, 412), (533, 409), (533, 406), (533, 404), (533, 401), (533, 398), (533, 396), (536, 393), (536, 392), (536, 389), (536, 385), (536, 384), (536, 381), (536, 378), (536, 376), (536, 373), (536, 371), (536, 368), (536, 365), (536, 364), (536, 360), (536, 357), (536, 356), (536, 353), (536, 351), (541, 351), (541, 348), (541, 345), (541, 343), (541, 340), (541, 339), (541, 335), (541, 332), (541, 331), (541, 328), (541, 324), (541, 323), (541, 320), (541, 318), (541, 315), (541, 312), (541, 310), (541, 307), (541, 304), (544, 303), (544, 299), (544, 298), (544, 295), (544, 292), (544, 290), (544, 287), (544, 284), (544, 282), (544, 279), (544, 278), (544, 274), (544, 271), (544, 270), (549, 270), (549, 267), (549, 264), (549, 262), (549, 259), (549, 257), (549, 254), (549, 251), (549, 249)]"314        },315        {316            "step": 21,317            "screenshot": "1084880108783314_21.png",318            "action": "CLICK",319            "info": [320                [321                    308,322                    542323                ],324                [325                    308,326                    542327                ]328            ],329            "ps": "[(308, 542)]"330        },331        {332            "step": 22,333            "screenshot": "1084880108783314_22.png",334            "action": "COMPLETE",335            "info": "",336            "ps": ""337        }338    ]339}