OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0954574276346528",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "WPS office",15 "Wikipedia",16 "PlantNet"17 ],18 "meta_task": "Find a photograph of a/an {}, identify it using a plant identifier app, and if the identification result is correct, record the image along with the plant name in a document.",19 "task": "Find a photograph of a/an Trumpet Vine, identify it using a plant identifier app, and if the identification result is correct, record the image along with the plant name in a document.",20 "instruction": "Locate a photograph of a Trumpet Vine, then use the PlantNet app to identify it. If the identification is correct, document the image and plant name using WPS Office and verify the information on Wikipedia."21 },22 "step_length": 18,23 "steps": [24 {25 "step": 0,26 "screenshot": "0954574276346528_0.png",27 "action": "CLICK",28 "info": [29 [30 372,31 24932 ],33 [34 372,35 24936 ]37 ],38 "ps": "[(372, 249)]"39 },40 {41 "step": 1,42 "screenshot": "0954574276346528_1.png",43 "action": "CLICK",44 "info": [45 [46 366,47 15648 ],49 [50 366,51 15652 ]53 ],54 "ps": "[(366, 156)]"55 },56 {57 "step": 2,58 "screenshot": "0954574276346528_2.png",59 "action": "TEXT",60 "info": "Trumpet Vine",61 "ps": ""62 },63 {64 "step": 3,65 "screenshot": "0954574276346528_3.png",66 "action": "CLICK",67 "info": [68 [69 502,70 16871 ],72 [73 502,74 16875 ]76 ],77 "ps": "[(502, 168)]"78 },79 {80 "step": 4,81 "screenshot": "0954574276346528_4.png",82 "action": "CLICK",83 "info": [84 [85 474,86 32287 ],88 [89 474,90 32291 ]92 ],93 "ps": "[(474, 322)]"94 },95 {96 "step": 5,97 "screenshot": "0954574276346528_5.png",98 "action": "CLICK",99 "info": [100 [101 938,102 97103 ],104 [105 938,106 97107 ]108 ],109 "ps": "[(938, 97)]"110 },111 {112 "step": 6,113 "screenshot": "0954574276346528_6.png",114 "action": "CLICK",115 "info": [116 [117 812,118 104119 ],120 [121 812,122 104123 ]124 ],125 "ps": "[(812, 104)]"126 },127 {128 "step": 7,129 "screenshot": "0954574276346528_7.png",130 "action": "CLICK",131 "info": [132 [133 673,134 107135 ],136 [137 673,138 107139 ]140 ],141 "ps": "[(673, 107)]"142 },143 {144 "step": 8,145 "screenshot": "0954574276346528_8.png",146 "action": "CLICK",147 "info": "KEY_HOME",148 "ps": ""149 },150 {151 "step": 9,152 "screenshot": "0954574276346528_9.png",153 "action": "CLICK",154 "info": [155 [156 617,157 383158 ],159 [160 617,161 382162 ]163 ],164 "ps": "[(617, 383), (617, 382)]"165 },166 {167 "step": 10,168 "screenshot": "0954574276346528_10.png",169 "action": "CLICK",170 "info": [171 [172 496,173 951174 ],175 [176 496,177 951178 ]179 ],180 "ps": "[(496, 951)]"181 },182 {183 "step": 11,184 "screenshot": "0954574276346528_11.png",185 "action": "CLICK",186 "info": [187 [188 152,189 592190 ],191 [192 152,193 592194 ]195 ],196 "ps": "[(152, 592)]"197 },198 {199 "step": 12,200 "screenshot": "0954574276346528_12.png",201 "action": "CLICK",202 "info": [203 [204 287,205 891206 ],207 [208 287,209 891210 ]211 ],212 "ps": "[(287, 891)]"213 },214 {215 "step": 13,216 "screenshot": "0954574276346528_13.png",217 "action": "CLICK",218 "info": [219 [220 381,221 395222 ],223 [224 381,225 395226 ]227 ],228 "ps": "[(381, 395)]"229 },230 {231 "step": 14,232 "screenshot": "0954574276346528_14.png",233 "action": "SCROLL",234 "info": [235 [236 412,237 912238 ],239 [240 544,241 203242 ]243 ],244 "ps": "[(412, 912), (412, 911), (412, 909), (412, 909), (412, 907), (412, 906), (411, 906), (411, 905), (411, 904), (411, 903), (411, 902), (411, 901), (408, 901), (408, 900), (408, 899), (408, 897), (408, 897), (408, 895), (408, 894), (408, 893), (408, 892), (408, 891), (408, 890), (408, 889), (408, 888), (405, 888), (405, 887), (405, 885), (405, 884), (405, 883), (405, 882), (405, 881), (405, 880), (405, 879), (405, 878), (405, 877), (405, 876), (405, 874), (405, 874), (405, 872), (405, 871), (405, 870), (405, 869), (405, 868), (405, 867), (405, 866), (405, 865), (405, 864), (405, 862), (405, 862), (405, 860), (405, 859), (405, 858), (405, 857), (405, 856), (405, 855), (405, 854), (405, 853), (405, 852), (405, 850), (405, 849), (405, 848), (405, 847), (405, 846), (405, 845), (405, 844), (405, 843), (405, 842), (405, 841), (405, 839), (405, 839), (405, 837), (405, 836), (405, 835), (405, 834), (405, 833), (405, 832), (405, 831), (405, 829), (405, 829), (405, 827), (405, 827), (405, 825), (405, 824), (405, 823), (405, 822), (405, 821), (408, 812), (408, 811), (408, 810), (408, 809), (408, 808), (408, 807), (408, 806), (411, 804), (411, 803), (411, 802), (411, 801), (411, 800), (411, 799), (411, 798), (411, 797), (411, 796), (411, 794), (411, 794), (411, 792), (412, 792), (412, 790), (412, 789), (412, 788), (412, 787), (412, 786), (412, 785), (412, 784), (412, 783), (412, 782), (412, 780), (412, 779), (412, 778), (412, 777), (412, 776), (412, 775), (412, 774), (412, 773), (412, 772), (415, 772), (415, 771), (415, 768), (415, 767), (415, 766), (415, 765), (415, 764), (415, 763), (415, 762), (417, 761), (417, 759), (417, 757), (417, 755), (417, 753), (417, 752), (417, 751), (417, 750), (417, 749), (420, 747), (420, 745), (420, 744), (420, 742), (420, 741), (420, 739), (423, 738), (423, 736), (423, 734), (423, 733), (423, 731), (423, 730), (423, 729), (423, 728), (424, 726), (424, 724), (424, 724), (424, 722), (424, 722), (424, 720), (424, 718), (424, 717), (424, 716), (427, 716), (427, 715), (427, 713), (427, 712), (427, 710), (427, 708), (430, 707), (430, 706), (430, 704), (430, 703), (430, 702), (430, 699), (430, 698), (430, 696), (430, 695), (430, 694), (430, 692), (432, 691), (432, 689), (432, 687), (432, 687), (432, 684), (432, 683), (432, 682), (435, 680), (435, 679), (435, 678), (435, 675), (435, 673), (437, 672), (437, 671), (437, 669), (437, 668), (437, 667), (437, 666), (437, 664), (437, 662), (439, 661), (439, 660), (439, 658), (439, 656), (442, 654), (442, 654), (442, 651), (442, 650), (442, 648), (442, 646), (442, 645), (442, 644), (442, 642), (442, 640), (444, 640), (444, 639), (444, 637), (444, 636), (444, 634), (444, 633), (444, 631), (444, 629), (444, 627), (444, 626), (444, 623), (447, 622), (447, 621), (447, 619), (447, 617), (447, 615), (447, 614), (449, 612), (449, 611), (449, 609), (449, 607), (449, 605), (449, 604), (449, 603), (449, 602), (451, 601), (451, 599), (451, 598), (451, 596), (451, 594), (454, 592), (454, 591), (454, 589), (454, 588), (454, 586), (454, 584), (454, 584), (454, 581), (454, 580), (457, 579), (457, 578), (457, 577), (457, 575), (457, 572), (457, 570), (457, 569), (459, 568), (459, 567), (459, 566), (459, 565), (459, 563), (459, 562), (459, 559), (459, 557), (462, 557), (462, 556), (462, 554), (462, 553), (462, 552), (462, 549), (462, 547), (462, 545), (462, 544), (462, 543), (462, 541), (462, 540), (462, 539), (466, 537), (466, 534), (466, 532), (466, 530), (466, 529), (466, 527), (466, 526), (466, 524), (466, 523), (466, 522), (466, 521), (466, 520), (469, 519), (469, 518), (469, 517), (469, 516), (469, 514), (469, 511), (471, 510), (471, 509), (471, 508), (471, 507), (471, 505), (474, 504), (474, 502), (474, 500), (474, 498), (476, 498), (476, 496), (476, 495), (476, 493), (476, 492), (478, 490), (478, 489), (478, 488), (478, 487), (481, 485), (481, 484), (481, 483), (481, 482), (481, 481), (483, 479), (483, 477), (483, 476), (483, 474), (483, 473), (486, 472), (486, 471), (486, 470), (486, 467), (486, 467), (486, 465), (486, 463), (486, 462), (488, 461), (488, 459), (488, 457), (488, 455), (488, 454), (488, 452), (490, 451), (490, 450), (490, 448), (490, 447), (490, 446), (490, 444), (493, 442), (493, 441), (493, 439), (493, 438), (493, 436), (496, 435), (496, 434), (496, 432), (496, 432), (496, 429), (496, 428), (496, 427), (496, 426), (496, 425), (496, 423), (498, 422), (498, 420), (498, 418), (498, 417), (498, 416), (498, 415), (498, 414), (498, 413), (500, 411), (500, 409), (500, 409), (500, 407), (500, 406), (500, 405), (500, 404), (500, 402), (500, 401), (500, 400), (500, 399), (500, 397), (500, 397), (500, 395), (500, 393), (500, 391), (500, 390), (500, 389), (502, 388), (502, 387), (502, 385), (502, 383), (502, 382), (502, 381), (502, 380), (502, 379), (502, 378), (505, 378), (505, 377), (505, 376), (505, 374), (505, 374), (505, 372), (505, 371), (505, 370), (508, 369), (508, 368), (508, 367), (508, 366), (508, 365), (508, 364), (508, 362), (508, 362), (508, 360), (508, 359), (510, 359), (510, 358), (510, 357), (510, 356), (510, 355), (510, 354), (510, 353), (512, 352), (512, 350), (512, 349), (512, 348), (512, 347), (512, 346), (512, 344), (515, 344), (515, 343), (515, 342), (515, 341), (515, 339), (515, 339), (515, 337), (515, 335), (515, 334), (515, 333), (515, 332), (515, 329), (517, 329), (517, 327), (517, 325), (517, 324), (517, 323), (517, 322), (520, 321), (520, 320), (520, 319), (520, 317), (520, 315), (520, 314), (522, 313), (522, 312), (522, 311), (522, 309), (522, 308), (522, 307), (524, 306), (524, 304), (524, 303), (524, 302), (524, 301), (524, 299), (524, 298), (527, 297), (527, 296), (527, 294), (527, 294), (527, 292), (527, 292), (527, 290), (527, 289), (529, 288), (529, 287), (529, 286), (529, 285), (529, 284), (529, 283), (529, 282), (529, 280), (529, 279), (529, 278), (529, 277), (529, 276), (529, 275), (532, 274), (532, 273), (532, 272), (532, 271), (532, 269), (532, 268), (532, 267), (535, 266), (535, 265), (535, 264), (535, 263), (535, 262), (535, 261), (537, 259), (537, 259), (537, 257), (537, 257), (537, 255), (537, 254), (537, 253), (537, 252), (537, 251), (537, 250), (537, 249), (539, 249), (539, 248), (539, 247), (539, 245), (539, 244), (539, 243), (539, 242), (539, 241), (541, 240), (541, 239), (541, 238), (541, 237), (541, 236), (541, 234), (541, 233), (541, 232), (541, 231), (541, 230), (541, 229), (541, 228), (541, 227), (541, 226), (544, 226), (544, 224), (544, 224), (544, 222), (544, 222), (544, 220), (544, 219), (544, 218), (544, 217), (544, 216), (544, 215), (544, 214), (544, 213), (544, 212), (544, 210), (544, 209), (544, 208), (544, 207), (544, 206), (544, 205), (544, 204), (544, 203)]"245 },246 {247 "step": 15,248 "screenshot": "0954574276346528_15.png",249 "action": "CLICK",250 "info": [251 [252 140,253 728254 ],255 [256 140,257 728258 ]259 ],260 "ps": "[(140, 728)]"261 },262 {263 "step": 16,264 "screenshot": "0954574276346528_16.png",265 "action": "CLICK",266 "info": [267 [268 719,269 508270 ],271 [272 719,273 508274 ]275 ],276 "ps": "[(719, 508)]"277 },278 {279 "step": 17,280 "screenshot": "0954574276346528_17.png",281 "action": "INCOMPLETE",282 "info": "",283 "ps": "recognition failed"284 }285 ]286}