OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "1127855290359296",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1840,8 "w": 2208,9 "device_name": "Pixel Fold"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Clock",15 "Chrome",16 "Messenger",17 "Ebay"18 ],19 "meta_task": "Plan a movie night by selecting a {} movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",20 "task": "Plan a movie night by selecting a science fiction movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",21 "instruction": "Organize a movie night by selecting a science fiction film on Chrome, adding some snacks to your cart on eBay, sending an invitation to Victor James via Messenger, and setting a reminder using the Clock app."22 },23 "step_length": 30,24 "steps": [25 {26 "step": 0,27 "screenshot": "1127855290359296_0.png",28 "action": "CLICK",29 "info": [30 [31 204,32 67633 ],34 [35 204,36 67637 ]38 ],39 "ps": "[(204, 676)]"40 },41 {42 "step": 1,43 "screenshot": "1127855290359296_1.png",44 "action": "CLICK",45 "info": [46 [47 255,48 40749 ],50 [51 255,52 40753 ]54 ],55 "ps": "[(255, 407)]"56 },57 {58 "step": 2,59 "screenshot": "1127855290359296_2.png",60 "action": "TEXT",61 "info": "science fiction movies",62 "ps": ""63 },64 {65 "step": 3,66 "screenshot": "1127855290359296_3.png",67 "action": "CLICK",68 "info": [69 [70 911,71 88172 ],73 [74 911,75 88176 ]77 ],78 "ps": "[(911, 881)]"79 },80 {81 "step": 4,82 "screenshot": "1127855290359296_4.png",83 "action": "SCROLL",84 "info": [85 [86 358,87 79188 ],89 [90 374,91 39592 ]93 ],94 "ps": "[(358, 791), (358, 790), (358, 787), (358, 783), (358, 780), (358, 776), (358, 773), (358, 770), (358, 766), (358, 763), (358, 759), (358, 756), (358, 753), (358, 748), (358, 746), (358, 741), (358, 739), (358, 736), (358, 732), (358, 729), (358, 724), (358, 722), (358, 719), (358, 715), (358, 711), (358, 707), (358, 704), (358, 702), (358, 697), (358, 695), (358, 689), (358, 684), (358, 682), (358, 679), (358, 674), (358, 667), (358, 656), (358, 647), (358, 638), (358, 623), (358, 612), (358, 601), (358, 590), (358, 585), (358, 579), (358, 573), (358, 567), (358, 564), (358, 558), (358, 555), (358, 553), (358, 549), (358, 546), (358, 545), (358, 542), (358, 538), (358, 535), (358, 532), (358, 526), (358, 521), (358, 515), (358, 512), (358, 509), (358, 503), (358, 498), (358, 492), (358, 489), (358, 488), (358, 484), (358, 478), (358, 472), (358, 470), (362, 470), (363, 470), (363, 466), (363, 462), (363, 457), (363, 454), (363, 451), (363, 447), (363, 444), (364, 442), (364, 441), (364, 440), (364, 435), (364, 432), (364, 429), (364, 425), (364, 422), (364, 418), (367, 415), (367, 414), (367, 413), (367, 408), (370, 407), (370, 404), (374, 399), (374, 395)]"95 },96 {97 "step": 5,98 "screenshot": "1127855290359296_5.png",99 "action": "CLICK",100 "info": [101 [102 149,103 599104 ],105 [106 149,107 599108 ]109 ],110 "ps": "[(149, 599)]"111 },112 {113 "step": 6,114 "screenshot": "1127855290359296_6.png",115 "action": "CLICK",116 "info": "KEY_HOME",117 "ps": ""118 },119 {120 "step": 7,121 "screenshot": "1127855290359296_7.png",122 "action": "CLICK",123 "info": [124 [125 533,126 492127 ],128 [129 533,130 492131 ]132 ],133 "ps": "[(533, 492)]"134 },135 {136 "step": 8,137 "screenshot": "1127855290359296_8.png",138 "action": "CLICK",139 "info": [140 [141 336,142 176143 ],144 [145 336,146 176147 ]148 ],149 "ps": "[(336, 176)]"150 },151 {152 "step": 9,153 "screenshot": "1127855290359296_9.png",154 "action": "TEXT",155 "info": "snacks",156 "ps": ""157 },158 {159 "step": 10,160 "screenshot": "1127855290359296_10.png",161 "action": "CLICK",162 "info": [163 [164 927,165 885166 ],167 [168 927,169 885170 ]171 ],172 "ps": "[(927, 885)]"173 },174 {175 "step": 11,176 "screenshot": "1127855290359296_11.png",177 "action": "CLICK",178 "info": [179 [180 420,181 274182 ],183 [184 420,185 274186 ]187 ],188 "ps": "[(420, 274)]"189 },190 {191 "step": 12,192 "screenshot": "1127855290359296_12.png",193 "action": "SCROLL",194 "info": [195 [196 243,197 769198 ],199 [200 297,201 93202 ]203 ],204 "ps": "[(243, 769), (243, 766), (243, 764), (243, 763), (243, 759), (243, 756), (243, 753), (243, 748), (245, 743), (245, 741), (245, 739), (247, 737), (247, 733), (247, 726), (247, 717), (247, 709), (247, 699), (247, 690), (247, 680), (247, 672), (247, 662), (247, 653), (247, 647), (247, 642), (247, 635), (247, 629), (248, 624), (248, 619), (250, 613), (250, 608), (250, 603), (250, 598), (250, 592), (250, 589), (251, 583), (251, 580), (251, 578), (251, 575), (251, 569), (251, 564), (251, 560), (251, 556), (253, 555), (253, 552), (253, 549), (253, 545), (253, 543), (253, 541), (253, 538), (253, 534), (253, 532), (253, 529), (253, 525), (253, 522), (253, 521), (253, 518), (253, 516), (256, 511), (258, 505), (259, 499), (261, 495), (261, 489), (261, 488), (262, 481), (264, 475), (264, 470), (264, 461), (264, 454), (264, 448), (264, 442), (264, 438), (264, 432), (264, 427), (264, 424), (264, 418), (264, 413), (264, 407), (264, 405), (264, 399), (264, 398), (264, 395), (267, 390), (267, 385), (267, 384), (267, 381), (267, 377), (267, 374), (267, 371), (267, 367), (267, 365), (269, 361), (269, 355), (269, 353), (269, 349), (269, 344), (269, 338), (269, 333), (269, 331), (269, 328), (269, 327), (269, 323), (269, 321), (269, 319), (269, 311), (269, 307), (272, 301), (273, 297), (273, 291), (273, 289), (273, 285), (273, 280), (273, 276), (273, 273), (273, 267), (273, 266), (273, 260), (273, 250), (273, 241), (273, 236), (273, 230), (273, 224), (273, 219), (273, 213), (273, 207), (273, 202), (273, 196), (273, 193), (273, 189), (273, 186), (273, 180), (273, 176), (273, 173), (276, 169), (276, 166), (276, 163), (279, 159), (279, 153), (279, 147), (281, 143), (281, 140), (281, 138), (283, 133), (283, 129), (286, 123), (286, 120), (288, 116), (288, 113), (288, 110), (292, 106), (292, 102), (295, 99), (297, 97), (297, 93)]"205 },206 {207 "step": 13,208 "screenshot": "1127855290359296_13.png",209 "action": "SCROLL",210 "info": [211 [212 406,213 669214 ],215 [216 409,217 296218 ]219 ],220 "ps": "[(406, 669), (402, 666), (401, 662), (400, 659), (400, 654), (400, 652), (396, 647), (396, 645), (396, 640), (396, 635), (396, 629), (396, 623), (396, 616), (396, 608), (396, 598), (396, 586), (392, 582), (388, 576), (388, 567), (388, 562), (388, 555), (388, 549), (388, 545), (388, 539), (388, 534), (388, 528), (388, 522), (388, 521), (388, 518), (388, 516), (388, 514), (388, 512), (388, 508), (388, 505), (388, 501), (388, 498), (388, 492), (388, 488), (388, 482), (388, 477), (388, 471), (388, 465), (388, 459), (388, 454), (388, 451), (388, 449), (388, 447), (388, 441), (392, 435), (392, 432), (392, 429), (392, 428), (392, 425), (392, 420), (392, 418), (392, 414), (392, 413), (392, 409), (392, 407), (392, 402), (392, 399), (392, 394), (392, 391), (392, 385), (395, 381), (395, 376), (395, 374), (395, 371), (395, 365), (398, 358), (398, 355), (400, 347), (402, 338), (402, 333), (403, 327), (404, 321), (404, 315), (407, 313), (407, 308), (407, 305), (408, 304), (409, 298), (409, 296)]"221 },222 {223 "step": 14,224 "screenshot": "1127855290359296_14.png",225 "action": "CLICK",226 "info": [227 [228 479,229 601230 ],231 [232 479,233 601234 ]235 ],236 "ps": "[(479, 601)]"237 },238 {239 "step": 15,240 "screenshot": "1127855290359296_15.png",241 "action": "CLICK",242 "info": "KEY_HOME",243 "ps": ""244 },245 {246 "step": 16,247 "screenshot": "1127855290359296_16.png",248 "action": "CLICK",249 "info": [250 [251 661,252 147253 ],254 [255 661,256 147257 ]258 ],259 "ps": "[(661, 147)]"260 },261 {262 "step": 17,263 "screenshot": "1127855290359296_17.png",264 "action": "CLICK",265 "info": [266 [267 453,268 556269 ],270 [271 453,272 556273 ]274 ],275 "ps": "[(453, 556)]"276 },277 {278 "step": 18,279 "screenshot": "1127855290359296_18.png",280 "action": "CLICK",281 "info": [282 [283 433,284 929285 ],286 [287 433,288 929289 ]290 ],291 "ps": "[(433, 929)]"292 },293 {294 "step": 19,295 "screenshot": "1127855290359296_19.png",296 "action": "TEXT",297 "info": "Do you want to see a movie with me this Sunday night?",298 "ps": ""299 },300 {301 "step": 20,302 "screenshot": "1127855290359296_20.png",303 "action": "CLICK",304 "info": [305 [306 764,307 496308 ],309 [310 764,311 496312 ]313 ],314 "ps": "[(764, 496)]"315 },316 {317 "step": 21,318 "screenshot": "1127855290359296_21.png",319 "action": "CLICK",320 "info": "KEY_HOME",321 "ps": ""322 },323 {324 "step": 22,325 "screenshot": "1127855290359296_22.png",326 "action": "CLICK",327 "info": [328 [329 427,330 491331 ],332 [333 427,334 491335 ]336 ],337 "ps": "[(427, 491)]"338 },339 {340 "step": 23,341 "screenshot": "1127855290359296_23.png",342 "action": "CLICK",343 "info": [344 [345 781,346 827347 ],348 [349 781,350 827351 ]352 ],353 "ps": "[(781, 827)]"354 },355 {356 "step": 24,357 "screenshot": "1127855290359296_24.png",358 "action": "CLICK",359 "info": [360 [361 346,362 560363 ],364 [365 346,366 560367 ]368 ],369 "ps": "[(346, 560)]"370 },371 {372 "step": 25,373 "screenshot": "1127855290359296_25.png",374 "action": "CLICK",375 "info": [376 [377 669,378 603379 ],380 [381 669,382 603383 ]384 ],385 "ps": "[(669, 603)]"386 },387 {388 "step": 26,389 "screenshot": "1127855290359296_26.png",390 "action": "CLICK",391 "info": [392 [393 548,394 461395 ],396 [397 548,398 461399 ]400 ],401 "ps": "[(548, 461)]"402 },403 {404 "step": 27,405 "screenshot": "1127855290359296_27.png",406 "action": "CLICK",407 "info": [408 [409 806,410 720411 ],412 [413 806,414 720415 ]416 ],417 "ps": "[(806, 720)]"418 },419 {420 "step": 28,421 "screenshot": "1127855290359296_28.png",422 "action": "CLICK",423 "info": [424 [425 631,426 408427 ],428 [429 631,430 408431 ]432 ],433 "ps": "[(631, 408)]"434 },435 {436 "step": 29,437 "screenshot": "1127855290359296_29.png",438 "action": "COMPLETE",439 "info": "",440 "ps": "Victor James"441 }442 ]443}