OpenGVLab/GUI-Odyssey

1{2    "episode_id": "0142794215333019",3    "device_info": {4        "product": "sdk_gphone64_x86_64",5        "release_version": "14",6        "sdk_version": "34",7        "h": 1600,8        "w": 2560,9        "device_name": "Pixel Tablet"10    },11    "task_info": {12        "category": "Multi_Apps",13        "app": [14            "Youtube",15            "Firefox Browser",16            "Google Docs"17        ],18        "meta_task": "Select one healthy lunch plan for next day, take a note and watch a video about how to make one of dishes.",19        "task": "Select one healthy lunch plan for next day, take a note and watch a video about how to make one of dishes.",20        "instruction": "Choose a healthy lunch plan for the next day, record your selection in Google Docs, and watch a video on YouTube, using Firefox Browser, about how to prepare one of the dishes."21    },22    "step_length": 20,23    "steps": [24        {25            "step": 0,26            "screenshot": "0142794215333019_0.png",27            "action": "CLICK",28            "info": [29                [30                    130,31                    40732                ],33                [34                    130,35                    40736                ]37            ],38            "ps": "[(130, 407)]"39        },40        {41            "step": 1,42            "screenshot": "0142794215333019_1.png",43            "action": "CLICK",44            "info": [45                [46                    160,47                    7248                ],49                [50                    160,51                    7252                ]53            ],54            "ps": "[(160, 72)]"55        },56        {57            "step": 2,58            "screenshot": "0142794215333019_2.png",59            "action": "TEXT",60            "info": "Healthy lunch plan",61            "ps": ""62        },63        {64            "step": 3,65            "screenshot": "0142794215333019_3.png",66            "action": "CLICK",67            "info": [68                [69                    916,70                    70371                ],72                [73                    916,74                    70375                ]76            ],77            "ps": "[(916, 703)]"78        },79        {80            "step": 4,81            "screenshot": "0142794215333019_4.png",82            "action": "SCROLL",83            "info": [84                [85                    286,86                    87487                ],88                [89                    322,90                    9291                ]92            ],93            "ps": "[(286, 874), (287, 874), (287, 873), (287, 873), (287, 872), (287, 871), (287, 869), (287, 868), (287, 867), (287, 866), (287, 864), (288, 863), (288, 862), (288, 861), (288, 859), (288, 858), (288, 857), (288, 856), (288, 854), (289, 853), (289, 852), (289, 851), (289, 849), (289, 848), (289, 847), (289, 846), (289, 844), (289, 843), (289, 842), (289, 841), (289, 839), (290, 838), (290, 837), (290, 836), (290, 834), (290, 833), (290, 832), (290, 831), (290, 829), (290, 828), (291, 827), (291, 826), (291, 824), (291, 823), (291, 822), (292, 822), (292, 821), (292, 819), (292, 818), (292, 816), (293, 813), (293, 811), (293, 809), (294, 808), (294, 807), (294, 806), (294, 804), (294, 803), (294, 802), (294, 801), (294, 799), (294, 798), (294, 797), (294, 796), (294, 794), (294, 792), (294, 791), (294, 789), (294, 788), (294, 786), (295, 784), (295, 783), (295, 781), (295, 779), (295, 778), (296, 777), (296, 774), (296, 773), (296, 772), (296, 771), (296, 769), (296, 768), (297, 766), (297, 764), (297, 763), (297, 761), (297, 759), (297, 757), (298, 756), (298, 754), (298, 753), (298, 752), (298, 751), (298, 748), (298, 747), (298, 744), (298, 743), (298, 743), (298, 741), (298, 738), (298, 737), (298, 736), (298, 734), (298, 733), (298, 732), (299, 731), (299, 728), (299, 726), (299, 724), (299, 723), (299, 722), (299, 721), (300, 719), (300, 718), (300, 716), (300, 714), (300, 713), (300, 712), (300, 709), (300, 708), (300, 707), (300, 704), (300, 703), (300, 699), (300, 698), (301, 696), (301, 694), (301, 693), (301, 692), (301, 691), (301, 689), (301, 687), (301, 686), (301, 684), (301, 682), (301, 681), (301, 679), (301, 678), (301, 677), (301, 676), (301, 674), (301, 673), (302, 672), (302, 671), (302, 668), (302, 666), (302, 664), (302, 662), (303, 661), (303, 659), (303, 658), (303, 656), (303, 653), (303, 652), (303, 651), (303, 648), (303, 647), (303, 646), (304, 646), (304, 643), (304, 642), (304, 641), (304, 639), (304, 637), (304, 636), (304, 634), (304, 632), (304, 631), (304, 628), (304, 627), (304, 624), (304, 623), (305, 622), (305, 619), (305, 618), (305, 617), (305, 616), (305, 613), (305, 612), (305, 611), (305, 609), (305, 608), (305, 607), (305, 604), (305, 603), (305, 601), (305, 599), (305, 598), (305, 597), (305, 594), (305, 592), (305, 589), (305, 587), (305, 586), (305, 583), (305, 582), (305, 579), (305, 578), (305, 576), (305, 573), (305, 572), (305, 569), (305, 567), (305, 564), (305, 562), (305, 559), (305, 557), (305, 554), (305, 552), (305, 549), (305, 546), (305, 543), (305, 541), (305, 538), (305, 537), (305, 534), (305, 533), (305, 531), (305, 528), (305, 526), (305, 523), (305, 521), (305, 518), (305, 516), (305, 513), (305, 512), (305, 508), (305, 507), (305, 504), (305, 503), (305, 501), (305, 498), (305, 496), (305, 493), (305, 491), (305, 488), (305, 486), (305, 483), (305, 481), (305, 478), (305, 476), (305, 473), (305, 471), (305, 468), (305, 466), (305, 463), (305, 461), (305, 459), (307, 454), (307, 452), (307, 449), (307, 448), (307, 446), (307, 443), (307, 441), (307, 438), (307, 436), (307, 434), (307, 432), (307, 429), (307, 427), (307, 424), (307, 423), (308, 421), (308, 418), (308, 416), (308, 413), (308, 407), (308, 404), (308, 402), (308, 398), (308, 396), (308, 393), (308, 391), (308, 389), (308, 388), (308, 383), (308, 382), (308, 379), (308, 374), (308, 373), (308, 371), (309, 368), (309, 363), (309, 359), (309, 357), (309, 353), (309, 351), (309, 348), (309, 344), (310, 342), (310, 339), (310, 337), (310, 333), (310, 331), (310, 328), (310, 324), (310, 322), (311, 319), (311, 313), (311, 311), (311, 309), (312, 304), (312, 302), (312, 299), (312, 297), (312, 296), (312, 291), (312, 288), (312, 286), (312, 283), (312, 279), (312, 277), (312, 274), (313, 272), (313, 269), (313, 266), (313, 263), (314, 261), (314, 258), (314, 256), (314, 254), (314, 252), (314, 249), (314, 247), (315, 244), (315, 242), (315, 239), (315, 237), (315, 236), (315, 233), (315, 231), (315, 228), (315, 226), (315, 223), (315, 221), (315, 218), (315, 216), (315, 213), (316, 212), (316, 209), (316, 207), (316, 204), (316, 203), (316, 201), (316, 199), (316, 197), (317, 197), (317, 194), (317, 192), (317, 191), (317, 189), (317, 187), (317, 186), (318, 183), (318, 182), (318, 181), (318, 178), (318, 177), (319, 174), (319, 173), (319, 172), (319, 171), (319, 169), (319, 168), (319, 167), (319, 164), (319, 162), (319, 159), (319, 157), (319, 156), (319, 154), (319, 153), (319, 152), (319, 151), (319, 149), (319, 147), (319, 146), (319, 144), (319, 143), (319, 142), (319, 141), (319, 139), (320, 138), (320, 137), (320, 136), (320, 134), (320, 132), (320, 129), (320, 128), (320, 127), (320, 126), (321, 124), (321, 123), (321, 122), (321, 121), (321, 119), (321, 118), (321, 117), (321, 116), (321, 114), (321, 113), (321, 112), (321, 111), (321, 109), (321, 108), (321, 107), (321, 106), (322, 104), (322, 103), (322, 102), (322, 101), (322, 99), (322, 98), (322, 97), (322, 96), (322, 94), (322, 93), (322, 92)]"94        },95        {96            "step": 5,97            "screenshot": "0142794215333019_5.png",98            "action": "CLICK",99            "info": [100                [101                    116,102                    847103                ],104                [105                    116,106                    847107                ]108            ],109            "ps": "[(116, 847)]"110        },111        {112            "step": 6,113            "screenshot": "0142794215333019_6.png",114            "action": "SCROLL",115            "info": [116                [117                    444,118                    888119                ],120                [121                    444,122                    91123                ]124            ],125            "ps": "[(444, 888), (444, 887), (445, 887), (445, 886), (445, 884), (445, 883), (445, 882), (445, 881), (445, 879), (445, 878), (445, 877), (445, 876), (445, 874), (445, 873), (445, 872), (445, 871), (445, 869), (445, 868), (445, 867), (445, 866), (445, 864), (445, 863), (445, 862), (445, 861), (445, 859), (445, 858), (445, 857), (445, 856), (445, 853), (445, 852), (446, 852), (446, 851), (446, 849), (446, 848), (446, 847), (446, 846), (446, 844), (446, 843), (446, 842), (446, 841), (446, 839), (446, 838), (447, 837), (447, 836), (447, 834), (447, 832), (447, 831), (447, 829), (447, 828), (447, 827), (448, 826), (448, 824), (448, 822), (448, 821), (448, 819), (448, 818), (448, 817), (448, 816), (448, 814), (448, 812), (448, 811), (449, 809), (449, 808), (449, 806), (449, 804), (449, 803), (449, 801), (449, 799), (449, 798), (449, 796), (449, 794), (450, 793), (450, 792), (450, 791), (450, 789), (450, 788), (450, 787), (450, 786), (451, 784), (451, 783), (451, 782), (451, 781), (451, 779), (451, 778), (451, 776), (451, 774), (451, 773), (451, 772), (452, 771), (452, 769), (452, 767), (452, 766), (453, 764), (453, 763), (453, 762), (453, 761), (453, 759), (453, 757), (454, 756), (454, 754), (454, 753), (454, 752), (454, 751), (454, 749), (454, 748), (454, 747), (454, 746), (454, 744), (454, 743), (454, 741), (455, 739), (455, 738), (455, 737), (455, 736), (455, 734), (455, 733), (455, 732), (455, 731), (455, 729), (455, 728), (455, 727), (455, 726), (456, 724), (456, 723), (456, 722), (456, 721), (456, 719), (456, 718), (456, 717), (456, 716), (457, 714), (457, 713), (457, 712), (457, 711), (457, 709), (457, 708), (457, 707), (457, 706), (457, 704), (458, 703), (458, 702), (458, 701), (458, 699), (458, 698), (458, 697), (458, 696), (458, 694), (458, 693), (458, 692), (458, 691), (458, 689), (458, 688), (458, 686), (458, 684), (458, 683), (458, 682), (458, 681), (458, 679), (458, 678), (458, 677), (458, 676), (458, 676), (458, 674), (458, 673), (458, 672), (458, 671), (458, 668), (458, 667), (458, 666), (458, 664), (458, 663), (458, 661), (458, 659), (459, 658), (459, 657), (459, 656), (459, 653), (459, 652), (459, 651), (459, 649), (459, 648), (459, 646), (459, 644), (459, 643), (459, 642), (459, 641), (459, 639), (459, 638), (459, 636), (459, 634), (459, 633), (459, 631), (459, 629), (459, 628), (459, 626), (459, 623), (460, 623), (460, 621), (460, 619), (460, 617), (460, 616), (460, 613), (460, 611), (460, 609), (460, 607), (460, 606), (460, 603), (460, 601), (460, 597), (460, 596), (460, 594), (460, 592), (460, 589), (460, 587), (460, 586), (460, 583), (460, 581), (460, 579), (460, 578), (460, 576), (460, 573), (460, 571), (460, 568), (460, 566), (460, 563), (460, 561), (460, 558), (458, 504), (458, 502), (458, 499), (458, 497), (458, 494), (458, 492), (458, 488), (458, 486), (458, 482), (458, 479), (458, 477), (458, 474), (458, 472), (458, 469), (458, 466), (458, 463), (458, 458), (458, 456), (458, 453), (458, 451), (458, 448), (457, 444), (457, 442), (457, 439), (457, 438), (457, 434), (457, 432), (457, 429), (457, 427), (457, 424), (457, 422), (457, 421), (457, 416), (457, 412), (457, 409), (457, 407), (457, 404), (457, 402), (457, 398), (457, 396), (457, 393), (457, 391), (457, 389), (456, 384), (456, 382), (456, 379), (456, 377), (456, 373), (456, 371), (456, 368), (456, 367), (456, 364), (456, 362), (456, 359), (456, 356), (456, 353), (456, 351), (456, 348), (456, 344), (456, 342), (456, 339), (456, 337), (456, 334), (456, 332), (456, 328), (456, 326), (455, 323), (455, 321), (455, 319), (455, 317), (455, 314), (455, 312), (455, 309), (455, 308), (455, 306), (455, 303), (455, 301), (455, 297), (455, 296), (455, 293), (455, 291), (455, 288), (455, 286), (455, 283), (455, 282), (455, 281), (455, 278), (455, 276), (455, 274), (454, 272), (454, 271), (454, 268), (454, 266), (454, 263), (454, 261), (454, 259), (453, 257), (453, 254), (453, 253), (453, 252), (453, 251), (453, 249), (453, 247), (453, 246), (453, 243), (453, 242), (453, 239), (452, 238), (452, 237), (452, 234), (452, 233), (452, 231), (452, 229), (452, 227), (452, 226), (452, 224), (452, 223), (452, 222), (452, 219), (451, 219), (451, 217), (451, 216), (451, 214), (451, 213), (451, 212), (451, 211), (451, 209), (451, 208), (451, 207), (451, 207), (451, 206), (451, 203), (451, 202), (451, 201), (451, 199), (451, 198), (451, 197), (451, 196), (451, 194), (451, 193), (451, 192), (451, 191), (451, 189), (451, 188), (451, 187), (451, 186), (451, 184), (451, 183), (451, 182), (451, 181), (451, 179), (451, 178), (451, 177), (451, 176), (451, 174), (451, 173), (451, 172), (451, 171), (451, 169), (451, 168), (451, 167), (450, 166), (450, 164), (450, 163), (450, 162), (450, 161), (450, 159), (450, 158), (450, 157), (450, 156), (449, 156), (449, 154), (449, 153), (449, 152), (449, 149), (449, 148), (448, 147), (448, 146), (448, 144), (448, 143), (448, 142), (448, 141), (448, 139), (448, 138), (448, 137), (448, 136), (447, 134), (447, 133), (447, 132), (447, 131), (447, 129), (447, 128), (447, 127), (447, 126), (446, 126), (446, 124), (446, 123), (446, 122), (446, 121), (446, 119), (446, 118), (446, 117), (446, 116), (446, 114), (446, 113), (446, 112), (446, 111), (445, 109), (445, 108), (445, 107), (445, 106), (445, 104), (445, 103), (444, 102), (444, 101), (444, 99), (444, 98), (444, 97), (444, 96), (444, 94), (444, 94), (444, 93), (444, 92), (444, 91)]"126        },127        {128            "step": 7,129            "screenshot": "0142794215333019_7.png",130            "action": "SCROLL",131            "info": [132                [133                    470,134                    881135                ],136                [137                    462,138                    146139                ]140            ],141            "ps": "[(470, 881), (471, 881), (471, 879), (471, 878), (471, 877), (471, 876), (471, 874), (471, 873), (471, 872), (471, 871), (471, 869), (471, 868), (471, 867), (471, 866), (471, 864), (471, 863), (471, 862), (471, 861), (471, 859), (471, 857), (471, 856), (471, 854), (471, 853), (471, 852), (471, 851), (471, 849), (471, 848), (471, 847), (471, 846), (471, 844), (471, 843), (471, 842), (471, 841), (471, 838), (471, 837), (470, 837), (470, 836), (470, 833), (470, 831), (470, 829), (470, 828), (470, 827), (470, 826), (470, 824), (470, 823), (470, 821), (470, 819), (469, 818), (469, 817), (469, 816), (469, 813), (469, 811), (469, 809), (469, 808), (469, 806), (469, 806), (469, 803), (469, 802), (469, 798), (469, 797), (469, 794), (469, 793), (469, 791), (469, 788), (469, 787), (468, 786), (468, 783), (468, 782), (468, 781), (468, 779), (468, 777), (468, 776), (468, 773), (468, 772), (468, 771), (468, 769), (468, 767), (468, 766), (468, 763), (468, 762), (468, 761), (468, 759), (468, 758), (468, 756), (468, 753), (468, 751), (468, 749), (468, 747), (468, 746), (468, 743), (468, 742), (468, 741), (468, 739), (468, 737), (468, 736), (468, 734), (468, 732), (468, 731), (468, 728), (468, 726), (468, 723), (468, 722), (468, 721), (468, 718), (468, 717), (468, 714), (468, 713), (468, 711), (468, 709), (468, 708), (468, 707), (468, 706), (468, 703), (468, 702), (468, 701), (468, 698), (468, 696), (468, 694), (468, 692), (468, 691), (468, 688), (468, 687), (468, 684), (468, 682), (468, 681), (468, 678), (468, 676), (468, 674), (468, 672), (468, 671), (468, 667), (468, 664), (468, 663), (468, 662), (468, 661), (468, 658), (468, 657), (468, 654), (468, 652), (468, 651), (468, 649), (468, 648), (468, 646), (468, 643), (468, 642), (468, 639), (468, 637), (468, 634), (468, 633), (468, 631), (468, 628), (468, 627), (468, 626), (468, 624), (468, 622), (468, 621), (468, 618), (468, 616), (468, 614), (468, 613), (468, 611), (468, 608), (468, 607), (468, 604), (468, 603), (468, 601), (468, 599), (468, 598), (468, 596), (468, 594), (468, 591), (468, 588), (468, 587), (468, 586), (468, 583), (468, 581), (468, 578), (468, 577), (468, 574), (468, 573), (468, 571), (468, 569), (468, 567), (468, 564), (468, 562), (468, 559), (468, 558), (468, 556), (468, 553), (468, 551), (468, 548), (468, 547), (468, 544), (468, 543), (468, 541), (468, 538), (468, 536), (468, 534), (468, 532), (468, 531), (468, 528), (468, 526), (468, 523), (468, 521), (468, 518), (468, 516), (468, 513), (468, 509), (468, 507), (468, 504), (468, 502), (468, 499), (468, 497), (468, 494), (468, 491), (468, 488), (468, 486), (468, 483), (468, 479), (468, 477), (468, 474), (468, 472), (468, 471), (468, 468), (468, 466), (468, 461), (468, 459), (468, 456), (468, 454), (468, 452), (468, 449), (468, 446), (468, 443), (468, 441), (468, 438), (468, 436), (468, 433), (468, 431), (468, 428), (468, 424), (468, 422), (468, 419), (468, 414), (468, 413), (468, 411), (468, 408), (468, 404), (468, 402), (468, 399), (468, 397), (468, 392), (468, 391), (468, 388), (468, 386), (468, 383), (468, 379), (468, 377), (468, 374), (468, 372), (467, 366), (467, 364), (467, 359), (467, 358), (467, 357), (467, 352), (467, 351), (467, 348), (467, 346), (467, 344), (467, 342), (467, 339), (467, 337), (467, 334), (467, 332), (467, 329), (467, 328), (467, 326), (467, 324), (467, 321), (467, 318), (467, 317), (467, 314), (467, 312), (467, 309), (467, 307), (467, 304), (467, 302), (466, 299), (466, 297), (466, 296), (466, 293), (466, 291), (466, 288), (466, 286), (466, 284), (466, 282), (466, 281), (466, 278), (466, 277), (466, 274), (466, 272), (466, 271), (466, 268), (466, 267), (466, 266), (466, 264), (466, 263), (466, 261), (466, 259), (466, 257), (466, 256), (466, 254), (466, 253), (466, 252), (466, 251), (466, 248), (466, 247), (466, 246), (466, 243), (466, 242), (466, 241), (466, 239), (466, 238), (466, 237), (466, 236), (466, 234), (466, 233), (466, 232), (466, 231), (466, 229), (466, 228), (466, 227), (466, 226), (466, 224), (466, 223), (466, 222), (466, 219), (465, 217), (465, 216), (465, 214), (465, 213), (465, 212), (465, 211), (465, 208), (465, 207), (465, 204), (465, 203), (465, 202), (465, 201), (465, 199), (465, 198), (465, 197), (465, 196), (465, 194), (465, 193), (465, 192), (465, 191), (465, 189), (465, 188), (465, 187), (464, 186), (464, 184), (464, 183), (464, 182), (464, 181), (464, 179), (464, 178), (464, 177), (464, 176), (464, 174), (463, 174), (463, 173), (463, 172), (463, 171), (463, 169), (463, 168), (463, 167), (463, 166), (463, 164), (462, 164), (462, 163), (462, 162), (462, 161), (462, 159), (462, 158), (462, 157), (462, 156), (462, 154), (462, 153), (462, 152), (462, 151), (462, 149), (462, 148), (462, 147), (462, 146)]"142        },143        {144            "step": 8,145            "screenshot": "0142794215333019_8.png",146            "action": "CLICK",147            "info": "KEY_HOME",148            "ps": ""149        },150        {151            "step": 9,152            "screenshot": "0142794215333019_9.png",153            "action": "CLICK",154            "info": [155                [156                    288,157                    564158                ],159                [160                    288,161                    564162                ]163            ],164            "ps": "[(288, 564)]"165        },166        {167            "step": 10,168            "screenshot": "0142794215333019_10.png",169            "action": "CLICK",170            "info": [171                [172                    959,173                    904174                ],175                [176                    959,177                    904178                ]179            ],180            "ps": "[(959, 904)]"181        },182        {183            "step": 11,184            "screenshot": "0142794215333019_11.png",185            "action": "CLICK",186            "info": [187                [188                    959,189                    842190                ],191                [192                    959,193                    842194                ]195            ],196            "ps": "[(959, 842)]"197        },198        {199            "step": 12,200            "screenshot": "0142794215333019_12.png",201            "action": "TEXT",202            "info": "Diet cola chicken",203            "ps": ""204        },205        {206            "step": 13,207            "screenshot": "0142794215333019_13.png",208            "action": "CLICK",209            "info": "KEY_HOME",210            "ps": ""211        },212        {213            "step": 14,214            "screenshot": "0142794215333019_14.png",215            "action": "CLICK",216            "info": [217                [218                    279,219                    726220                ],221                [222                    279,223                    726224                ]225            ],226            "ps": "[(279, 726)]"227        },228        {229            "step": 15,230            "screenshot": "0142794215333019_15.png",231            "action": "CLICK",232            "info": [233                [234                    976,235                    56236                ],237                [238                    976,239                    56240                ]241            ],242            "ps": "[(976, 56)]"243        },244        {245            "step": 16,246            "screenshot": "0142794215333019_16.png",247            "action": "TEXT",248            "info": "how to make diet cola chicken",249            "ps": ""250        },251        {252            "step": 17,253            "screenshot": "0142794215333019_17.png",254            "action": "CLICK",255            "info": [256                [257                    904,258                    692259                ],260                [261                    904,262                    692263                ]264            ],265            "ps": "[(904, 692)]"266        },267        {268            "step": 18,269            "screenshot": "0142794215333019_18.png",270            "action": "CLICK",271            "info": [272                [273                    465,274                    437275                ],276                [277                    465,278                    437279                ]280            ],281            "ps": "[(465, 437)]"282        },283        {284            "step": 19,285            "screenshot": "0142794215333019_19.png",286            "action": "COMPLETE",287            "info": "",288            "ps": ""289        }290    ]291}