OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0456589240667140",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Weather & Radar",15 "Gmail",16 "Mapillary"17 ],18 "meta_task": "Look for a new hiking trail, check the weather forecast for the weekend and invite friends to join.",19 "task": "Look for a new hiking trail, check the weather forecast for the weekend and invite friends to join.",20 "instruction": "Use Mapillary to discover a new hiking trail, then check the weekend weather forecast with Weather & Radar. Finally, send an invitation to caba62244@gmail.com via Gmail to join the hike."21 },22 "step_length": 30,23 "steps": [24 {25 "step": 0,26 "screenshot": "0456589240667140_0.png",27 "action": "CLICK",28 "info": [29 [30 136,31 65232 ],33 [34 136,35 65236 ]37 ],38 "ps": "[(136, 652)]"39 },40 {41 "step": 1,42 "screenshot": "0456589240667140_1.png",43 "action": "CLICK",44 "info": "KEY_HOME",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "0456589240667140_2.png",50 "action": "CLICK",51 "info": [52 [53 136,54 39055 ],56 [57 136,58 39059 ]60 ],61 "ps": "[(136, 390)]"62 },63 {64 "step": 3,65 "screenshot": "0456589240667140_3.png",66 "action": "CLICK",67 "info": [68 [69 517,70 94271 ],72 [73 517,74 94275 ]76 ],77 "ps": "[(517, 942)]"78 },79 {80 "step": 4,81 "screenshot": "0456589240667140_4.png",82 "action": "CLICK",83 "info": [84 [85 785,86 92687 ],88 [89 785,90 92691 ]92 ],93 "ps": "[(785, 926)]"94 },95 {96 "step": 5,97 "screenshot": "0456589240667140_5.png",98 "action": "CLICK",99 "info": [100 [101 459,102 70103 ],104 [105 459,106 70107 ]108 ],109 "ps": "[(459, 70)]"110 },111 {112 "step": 6,113 "screenshot": "0456589240667140_6.png",114 "action": "TEXT",115 "info": "hiking trail",116 "ps": ""117 },118 {119 "step": 7,120 "screenshot": "0456589240667140_7.png",121 "action": "CLICK",122 "info": [123 [124 678,125 226126 ],127 [128 678,129 226130 ]131 ],132 "ps": "[(678, 226)]"133 },134 {135 "step": 8,136 "screenshot": "0456589240667140_8.png",137 "action": "CLICK",138 "info": "KEY_HOME",139 "ps": ""140 },141 {142 "step": 9,143 "screenshot": "0456589240667140_9.png",144 "action": "CLICK",145 "info": [146 [147 578,148 504149 ],150 [151 578,152 504153 ]154 ],155 "ps": "[(578, 504)]"156 },157 {158 "step": 10,159 "screenshot": "0456589240667140_10.png",160 "action": "CLICK",161 "info": [162 [163 919,164 62165 ],166 [167 919,168 62169 ]170 ],171 "ps": "[(919, 62)]"172 },173 {174 "step": 11,175 "screenshot": "0456589240667140_11.png",176 "action": "CLICK",177 "info": [178 [179 288,180 138181 ],182 [183 288,184 138185 ]186 ],187 "ps": "[(288, 138)]"188 },189 {190 "step": 12,191 "screenshot": "0456589240667140_12.png",192 "action": "TEXT",193 "info": "Los Angeles",194 "ps": ""195 },196 {197 "step": 13,198 "screenshot": "0456589240667140_13.png",199 "action": "CLICK",200 "info": [201 [202 284,203 217204 ],205 [206 284,207 217208 ]209 ],210 "ps": "[(284, 217)]"211 },212 {213 "step": 14,214 "screenshot": "0456589240667140_14.png",215 "action": "SCROLL",216 "info": [217 [218 574,219 848220 ],221 [222 471,223 138224 ]225 ],226 "ps": "[(574, 848), (574, 846), (574, 843), (574, 841), (574, 839), (574, 835), (575, 831), (575, 827), (575, 824), (575, 820), (575, 815), (575, 811), (575, 807), (575, 800), (575, 794), (575, 789), (575, 785), (575, 780), (575, 776), (575, 772), (575, 767), (575, 765), (575, 761), (575, 757), (575, 752), (575, 748), (575, 747), (568, 742), (568, 740), (568, 737), (568, 736), (568, 733), (566, 729), (566, 724), (566, 720), (566, 716), (566, 715), (566, 713), (566, 709), (566, 705), (566, 701), (566, 698), (566, 694), (566, 689), (566, 685), (559, 681), (559, 680), (559, 677), (551, 674), (551, 671), (551, 667), (551, 662), (551, 658), (551, 654), (551, 651), (551, 650), (551, 647), (551, 642), (551, 638), (547, 634), (547, 633), (539, 628), (539, 626), (539, 624), (539, 621), (539, 616), (539, 614), (539, 613), (539, 611), (532, 607), (532, 603), (532, 600), (532, 596), (532, 591), (527, 587), (527, 586), (527, 582), (527, 569), (527, 565), (527, 563), (527, 558), (520, 555), (520, 554), (515, 541), (512, 524), (512, 516), (512, 507), (508, 488), (508, 471), (508, 463), (508, 457), (508, 451), (508, 447), (508, 442), (508, 438), (508, 434), (508, 428), (508, 420), (508, 418), (508, 417), (508, 414), (508, 409), (500, 407), (500, 406), (493, 403), (493, 399), (493, 397), (493, 393), (493, 389), (493, 384), (493, 370), (490, 367), (490, 366), (490, 364), (490, 362), (490, 360), (490, 359), (490, 357), (490, 355), (490, 353), (490, 349), (490, 347), (490, 345), (490, 342), (490, 339), (490, 339), (490, 335), (490, 333), (490, 331), (490, 327), (490, 323), (490, 322), (490, 320), (490, 317), (486, 314), (486, 310), (486, 309), (486, 306), (486, 303), (486, 301), (486, 298), (486, 296), (486, 292), (486, 290), (486, 289), (486, 288), (486, 287), (486, 285), (486, 282), (486, 279), (486, 275), (486, 271), (486, 267), (486, 266), (486, 264), (486, 262), (486, 259), (478, 257), (478, 252), (478, 248), (478, 244), (471, 242), (471, 239), (471, 237), (471, 236), (471, 232), (471, 230), (471, 228), (471, 224), (471, 222), (471, 219), (471, 217), (471, 213), (471, 210), (471, 209), (471, 206), (471, 204), (471, 201), (471, 196), (471, 194), (471, 192), (471, 189), (471, 187), (471, 183), (471, 179), (471, 177), (471, 172), (471, 170), (471, 167), (471, 164), (471, 161), (471, 159), (471, 157), (471, 154), (471, 151), (471, 150), (471, 149), (471, 148), (471, 145), (471, 144), (471, 142), (471, 142), (471, 140), (471, 138)]"227 },228 {229 "step": 15,230 "screenshot": "0456589240667140_15.png",231 "action": "SCROLL",232 "info": [233 [234 541,235 952236 ],237 [238 262,239 278240 ]241 ],242 "ps": "[(541, 952), (541, 950), (541, 947), (537, 944), (537, 944), (537, 939), (529, 939), (529, 937), (529, 934), (524, 934), (522, 932), (517, 928), (512, 926), (510, 923), (505, 919), (505, 917), (500, 914), (493, 913), (493, 909), (488, 909), (481, 905), (478, 903), (474, 900), (469, 897), (469, 894), (469, 892), (462, 889), (462, 888), (462, 883), (457, 883), (457, 881), (457, 879), (449, 879), (449, 878), (444, 874), (444, 872), (444, 869), (444, 867), (437, 867), (437, 865), (437, 862), (437, 859), (437, 856), (432, 854), (432, 850), (432, 848), (432, 845), (432, 843), (424, 839), (424, 839), (420, 835), (420, 834), (420, 833), (420, 829), (412, 827), (412, 823), (412, 819), (405, 815), (405, 813), (405, 808), (400, 806), (400, 804), (396, 801), (391, 797), (391, 794), (391, 790), (384, 787), (384, 783), (376, 779), (376, 777), (376, 773), (376, 766), (374, 762), (374, 761), (374, 759), (372, 754), (372, 750), (364, 745), (364, 741), (364, 740), (364, 737), (357, 734), (357, 730), (357, 727), (354, 722), (354, 720), (354, 716), (354, 713), (354, 708), (354, 704), (354, 703), (354, 701), (354, 698), (354, 694), (349, 691), (349, 687), (349, 684), (349, 680), (349, 677), (349, 671), (349, 667), (349, 662), (349, 658), (349, 654), (349, 649), (349, 645), (349, 640), (349, 637), (349, 633), (349, 621), (349, 617), (349, 613), (349, 609), (349, 607), (349, 603), (349, 599), (349, 594), (349, 590), (349, 588), (349, 587), (349, 582), (349, 578), (349, 574), (349, 572), (349, 570), (347, 566), (347, 562), (347, 558), (347, 556), (347, 552), (347, 549), (347, 545), (347, 542), (347, 540), (347, 535), (347, 532), (347, 528), (347, 523), (339, 521), (339, 518), (339, 516), (339, 514), (339, 510), (339, 508), (335, 505), (335, 502), (335, 498), (335, 495), (335, 493), (327, 493), (327, 492), (327, 489), (327, 486), (325, 485), (325, 483), (325, 479), (325, 477), (323, 475), (323, 473), (323, 471), (323, 470), (323, 467), (323, 463), (323, 459), (323, 458), (323, 454), (323, 452), (315, 449), (315, 447), (315, 444), (311, 441), (311, 440), (311, 438), (311, 435), (311, 434), (311, 432), (311, 427), (311, 424), (303, 422), (303, 420), (303, 417), (303, 415), (303, 413), (303, 409), (303, 405), (303, 403), (303, 402), (303, 400), (303, 397), (303, 395), (303, 393), (303, 390), (303, 388), (303, 385), (303, 382), (303, 380), (296, 376), (296, 371), (296, 368), (291, 366), (291, 362), (291, 360), (291, 357), (291, 355), (291, 352), (287, 350), (287, 349), (287, 347), (287, 344), (287, 343), (287, 342), (281, 339), (281, 336), (281, 334), (281, 332), (274, 329), (274, 327), (274, 324), (274, 323), (269, 321), (269, 319), (269, 318), (269, 314), (269, 312), (269, 309), (262, 309), (262, 307), (262, 304), (262, 301), (262, 300), (262, 299), (262, 297), (262, 294), (262, 292), (262, 289), (262, 286), (262, 284), (262, 280), (262, 278)]"243 },244 {245 "step": 16,246 "screenshot": "0456589240667140_16.png",247 "action": "SCROLL",248 "info": [249 [250 731,251 516252 ],253 [254 215,255 514256 ]257 ],258 "ps": "[(731, 516), (726, 516), (722, 516), (714, 516), (710, 516), (705, 516), (698, 516), (692, 516), (686, 516), (680, 516), (673, 514), (671, 514), (663, 514), (656, 514), (651, 514), (644, 514), (641, 514), (637, 514), (634, 514), (629, 514), (622, 514), (617, 514), (614, 514), (612, 514), (607, 514), (600, 514), (595, 514), (587, 514), (583, 514), (578, 514), (571, 514), (566, 514), (559, 514), (554, 514), (549, 514), (541, 514), (537, 514), (529, 514), (527, 514), (522, 514), (517, 514), (510, 514), (505, 514), (498, 514), (493, 514), (488, 514), (481, 514), (478, 514), (476, 514), (471, 514), (469, 514), (466, 514), (459, 514), (454, 514), (449, 514), (442, 514), (437, 514), (430, 514), (424, 514), (415, 514), (411, 514), (403, 514), (398, 514), (391, 514), (386, 514), (381, 514), (374, 514), (369, 514), (362, 514), (357, 514), (354, 514), (352, 514), (347, 514), (339, 514), (330, 514), (327, 514), (323, 514), (318, 514), (311, 514), (308, 514), (303, 514), (296, 514), (291, 514), (287, 514), (279, 514), (274, 514), (267, 514), (262, 514), (257, 514), (249, 514), (245, 514), (237, 514), (233, 514), (228, 514), (221, 514), (215, 514)]"259 },260 {261 "step": 17,262 "screenshot": "0456589240667140_17.png",263 "action": "SCROLL",264 "info": [265 [266 814,267 524268 ],269 [270 113,271 557272 ]273 ],274 "ps": "[(814, 524), (814, 522), (812, 522), (810, 522), (804, 522), (797, 522), (795, 523), (790, 523), (787, 523), (777, 524), (768, 526), (765, 526), (761, 526), (746, 526), (737, 528), (722, 529), (705, 529), (683, 531), (663, 532), (646, 532), (624, 534), (612, 535), (602, 537), (593, 537), (581, 539), (571, 539), (566, 539), (559, 542), (554, 542), (549, 542), (539, 542), (527, 545), (522, 545), (517, 545), (510, 545), (505, 546), (498, 546), (493, 546), (486, 547), (481, 547), (474, 547), (469, 547), (463, 547), (457, 549), (447, 549), (439, 552), (430, 552), (424, 552), (423, 553), (417, 553), (408, 553), (400, 553), (391, 553), (388, 553), (384, 553), (378, 553), (372, 556), (364, 556), (359, 556), (354, 556), (352, 557), (345, 557), (339, 557), (335, 557), (330, 557), (325, 557), (320, 557), (312, 557), (311, 557), (303, 557), (293, 557), (284, 557), (276, 557), (274, 557), (269, 557), (260, 557), (249, 557), (245, 557), (237, 557), (235, 557), (230, 557), (228, 557), (223, 557), (218, 557), (213, 557), (208, 557), (201, 557), (199, 557), (196, 557), (194, 557), (188, 557), (184, 557), (174, 557), (167, 557), (162, 557), (155, 557), (149, 557), (143, 557), (137, 557), (133, 557), (125, 557), (121, 557), (113, 557)]"275 },276 {277 "step": 18,278 "screenshot": "0456589240667140_18.png",279 "action": "CLICK",280 "info": [281 [282 345,283 532284 ],285 [286 345,287 532288 ]289 ],290 "ps": "[(345, 532)]"291 },292 {293 "step": 19,294 "screenshot": "0456589240667140_19.png",295 "action": "CLICK",296 "info": [297 [298 547,299 528300 ],301 [302 547,303 528304 ]305 ],306 "ps": "[(547, 528)]"307 },308 {309 "step": 20,310 "screenshot": "0456589240667140_20.png",311 "action": "CLICK",312 "info": "KEY_HOME",313 "ps": ""314 },315 {316 "step": 21,317 "screenshot": "0456589240667140_21.png",318 "action": "CLICK",319 "info": [320 [321 374,322 140323 ],324 [325 374,326 140327 ]328 ],329 "ps": "[(374, 140)]"330 },331 {332 "step": 22,333 "screenshot": "0456589240667140_22.png",334 "action": "CLICK",335 "info": [336 [337 862,338 876339 ],340 [341 862,342 876343 ]344 ],345 "ps": "[(862, 876)]"346 },347 {348 "step": 23,349 "screenshot": "0456589240667140_23.png",350 "action": "TEXT",351 "info": "caba62244@gmail.com",352 "ps": ""353 },354 {355 "step": 24,356 "screenshot": "0456589240667140_24.png",357 "action": "CLICK",358 "info": [359 [360 176,361 254362 ],363 [364 176,365 254366 ]367 ],368 "ps": "[(176, 254)]"369 },370 {371 "step": 25,372 "screenshot": "0456589240667140_25.png",373 "action": "CLICK",374 "info": [375 [376 230,377 320378 ],379 [380 230,381 320382 ]383 ],384 "ps": "[(230, 320)]"385 },386 {387 "step": 26,388 "screenshot": "0456589240667140_26.png",389 "action": "TEXT",390 "info": "Do you want to go to Los Angeles with me for a hiking trail next Sunday?",391 "ps": ""392 },393 {394 "step": 27,395 "screenshot": "0456589240667140_27.png",396 "action": "SCROLL",397 "info": [398 [399 602,400 105401 ],402 [403 673,404 0405 ]406 ],407 "ps": "[(602, 105), (602, 103), (602, 100), (602, 96), (602, 93), (602, 82), (602, 77), (607, 74), (610, 72), (610, 68), (612, 64), (612, 59), (617, 53), (624, 42), (632, 34), (632, 30), (639, 26), (639, 21), (656, 12), (663, 6), (673, 0)]"408 },409 {410 "step": 28,411 "screenshot": "0456589240667140_28.png",412 "action": "CLICK",413 "info": [414 [415 804,416 62417 ],418 [419 804,420 62421 ]422 ],423 "ps": "[(804, 62)]"424 },425 {426 "step": 29,427 "screenshot": "0456589240667140_29.png",428 "action": "COMPLETE",429 "info": "",430 "ps": "caba62244@gmail.com"431 }432 ]433}