OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0769779032616734",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1840,8 "w": 2208,9 "device_name": "Pixel Fold"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Chrome",15 "Clock",16 "Ebay",17 "X"18 ],19 "meta_task": "Plan a movie night by selecting a {} movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",20 "task": "Plan a movie night by selecting a documentary movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",21 "instruction": "Organize a movie night by choosing a documentary film using Chrome, adding snacks to your Ebay cart, sending invitations to liudehu19294094 via X, and setting a reminder on Clock."22 },23 "step_length": 30,24 "steps": [25 {26 "step": 0,27 "screenshot": "0769779032616734_0.png",28 "action": "CLICK",29 "info": [30 [31 185,32 64533 ],34 [35 185,36 64537 ]38 ],39 "ps": "[(185, 645)]"40 },41 {42 "step": 1,43 "screenshot": "0769779032616734_1.png",44 "action": "CLICK",45 "info": [46 [47 299,48 42549 ],50 [51 299,52 42553 ]54 ],55 "ps": "[(299, 425)]"56 },57 {58 "step": 2,59 "screenshot": "0769779032616734_2.png",60 "action": "TEXT",61 "info": "documentary movies",62 "ps": ""63 },64 {65 "step": 3,66 "screenshot": "0769779032616734_3.png",67 "action": "CLICK",68 "info": [69 [70 927,71 89772 ],73 [74 927,75 89776 ]77 ],78 "ps": "[(927, 897)]"79 },80 {81 "step": 4,82 "screenshot": "0769779032616734_4.png",83 "action": "SCROLL",84 "info": [85 [86 465,87 80788 ],89 [90 428,91 46292 ]93 ],94 "ps": "[(465, 807), (465, 805), (465, 804), (465, 800), (465, 794), (465, 789), (465, 783), (465, 777), (465, 771), (465, 767), (465, 761), (465, 756), (465, 753), (465, 747), (465, 739), (465, 733), (465, 730), (465, 726), (465, 724), (465, 720), (465, 715), (465, 710), (465, 704), (465, 699), (465, 696), (465, 693), (465, 689), (465, 688), (465, 684), (465, 680), (465, 677), (465, 676), (465, 673), (465, 669), (465, 666), (465, 660), (465, 659), (465, 656), (465, 651), (465, 645), (465, 640), (461, 636), (461, 633), (461, 630), (461, 626), (461, 623), (461, 619), (461, 616), (458, 612), (458, 609), (458, 608), (458, 605), (456, 601), (456, 599), (456, 596), (452, 589), (452, 588), (448, 583), (448, 579), (448, 576), (448, 575), (448, 572), (448, 567), (448, 562), (448, 556), (448, 555), (448, 552), (446, 548), (446, 546), (446, 543), (446, 539), (442, 536), (442, 535), (442, 530), (439, 528), (439, 522), (439, 518), (439, 512), (439, 511), (439, 507), (439, 501), (439, 496), (439, 494), (439, 489), (437, 489), (437, 486), (433, 482), (433, 479), (433, 477), (433, 472), (431, 472), (431, 470), (428, 470), (428, 465), (428, 462)]"95 },96 {97 "step": 5,98 "screenshot": "0769779032616734_5.png",99 "action": "CLICK",100 "info": [101 [102 253,103 635104 ],105 [106 253,107 635108 ]109 ],110 "ps": "[(253, 635)]"111 },112 {113 "step": 6,114 "screenshot": "0769779032616734_6.png",115 "action": "CLICK",116 "info": "KEY_HOME",117 "ps": ""118 },119 {120 "step": 7,121 "screenshot": "0769779032616734_7.png",122 "action": "CLICK",123 "info": [124 [125 555,126 475127 ],128 [129 555,130 475131 ]132 ],133 "ps": "[(555, 475)]"134 },135 {136 "step": 8,137 "screenshot": "0769779032616734_8.png",138 "action": "CLICK",139 "info": [140 [141 379,142 156143 ],144 [145 379,146 156147 ]148 ],149 "ps": "[(379, 156)]"150 },151 {152 "step": 9,153 "screenshot": "0769779032616734_9.png",154 "action": "TEXT",155 "info": "snacks",156 "ps": ""157 },158 {159 "step": 10,160 "screenshot": "0769779032616734_10.png",161 "action": "CLICK",162 "info": [163 [164 899,165 884166 ],167 [168 899,169 884170 ]171 ],172 "ps": "[(899, 884)]"173 },174 {175 "step": 11,176 "screenshot": "0769779032616734_11.png",177 "action": "CLICK",178 "info": [179 [180 827,181 284182 ],183 [184 827,185 284186 ]187 ],188 "ps": "[(827, 284)]"189 },190 {191 "step": 12,192 "screenshot": "0769779032616734_12.png",193 "action": "SCROLL",194 "info": [195 [196 611,197 876198 ],199 [200 613,201 421202 ]203 ],204 "ps": "[(611, 876), (607, 871), (607, 865), (607, 864), (607, 860), (607, 854), (604, 848), (604, 846), (604, 841), (605, 834), (604, 827), (604, 820), (604, 809), (604, 801), (604, 793), (604, 786), (604, 777), (604, 769), (604, 761), (605, 756), (605, 750), (605, 745), (605, 740), (603, 736), (603, 733), (603, 730), (603, 726), (601, 723), (601, 717), (601, 711), (601, 706), (601, 702), (601, 696), (601, 690), (601, 684), (601, 682), (601, 676), (601, 670), (601, 665), (601, 656), (601, 651), (604, 642), (606, 636), (606, 633), (609, 628), (609, 622), (610, 613), (610, 605), (610, 599), (610, 594), (610, 588), (610, 582), (610, 579), (610, 576), (610, 572), (610, 566), (610, 564), (610, 562), (610, 558), (610, 555), (610, 552), (610, 548), (610, 545), (610, 541), (610, 538), (610, 535), (610, 532), (610, 529), (610, 526), (610, 522), (610, 521), (610, 518), (610, 515), (610, 509), (610, 505), (610, 502), (610, 501), (610, 498), (610, 492), (610, 486), (610, 481), (610, 475), (613, 468), (613, 465), (613, 459), (613, 454), (613, 451), (613, 447), (613, 444), (613, 440), (613, 438), (613, 435), (613, 432), (613, 428), (613, 425), (613, 421)]"205 },206 {207 "step": 13,208 "screenshot": "0769779032616734_13.png",209 "action": "SCROLL",210 "info": [211 [212 617,213 816214 ],215 [216 607,217 269218 ]219 ],220 "ps": "[(617, 816), (617, 814), (617, 810), (617, 807), (618, 805), (618, 801), (622, 798), (622, 794), (622, 789), (622, 783), (622, 777), (622, 776), (622, 773), (622, 770), (622, 766), (622, 760), (622, 754), (622, 748), (622, 743), (622, 737), (622, 732), (622, 726), (622, 720), (622, 711), (618, 709), (618, 703), (618, 692), (618, 684), (619, 673), (619, 662), (619, 647), (619, 639), (618, 633), (615, 628), (615, 622), (615, 616), (615, 610), (615, 605), (615, 599), (615, 589), (615, 580), (615, 575), (615, 572), (615, 567), (612, 565), (612, 559), (609, 552), (609, 549), (609, 546), (609, 542), (609, 536), (609, 534), (609, 528), (609, 526), (609, 522), (607, 521), (607, 516), (607, 511), (607, 509), (607, 505), (607, 499), (607, 496), (607, 491), (607, 485), (607, 479), (607, 477), (607, 468), (607, 459), (607, 454), (607, 448), (607, 438), (607, 434), (607, 431), (607, 428), (607, 427), (607, 422), (607, 420), (607, 417), (607, 413), (607, 407), (607, 404), (607, 399), (607, 397), (607, 394), (607, 390), (607, 386), (607, 381), (607, 377), (607, 371), (607, 365), (607, 360), (607, 354), (607, 351), (607, 349), (607, 347), (607, 341), (607, 335), (607, 331), (607, 328), (607, 321), (607, 319), (607, 314), (607, 311), (607, 307), (607, 304), (607, 301), (607, 297), (607, 294), (607, 290), (607, 287), (607, 284), (607, 283), (607, 278), (607, 276), (607, 271), (607, 269)]"221 },222 {223 "step": 14,224 "screenshot": "0769779032616734_14.png",225 "action": "CLICK",226 "info": [227 [228 588,229 689230 ],231 [232 588,233 689234 ]235 ],236 "ps": "[(588, 689)]"237 },238 {239 "step": 15,240 "screenshot": "0769779032616734_15.png",241 "action": "CLICK",242 "info": "KEY_HOME",243 "ps": ""244 },245 {246 "step": 16,247 "screenshot": "0769779032616734_16.png",248 "action": "CLICK",249 "info": [250 [251 563,252 330253 ],254 [255 563,256 330257 ]258 ],259 "ps": "[(563, 330)]"260 },261 {262 "step": 17,263 "screenshot": "0769779032616734_17.png",264 "action": "CLICK",265 "info": [266 [267 904,268 913269 ],270 [271 904,272 913273 ]274 ],275 "ps": "[(904, 913)]"276 },277 {278 "step": 18,279 "screenshot": "0769779032616734_18.png",280 "action": "CLICK",281 "info": [282 [283 471,284 184285 ],286 [287 471,288 184289 ]290 ],291 "ps": "[(471, 184)]"292 },293 {294 "step": 19,295 "screenshot": "0769779032616734_19.png",296 "action": "CLICK",297 "info": [298 [299 214,300 907301 ],302 [303 214,304 907305 ]306 ],307 "ps": "[(214, 907)]"308 },309 {310 "step": 20,311 "screenshot": "0769779032616734_20.png",312 "action": "TEXT",313 "info": "Do you want to see a movie with me this Sunday night?",314 "ps": ""315 },316 {317 "step": 21,318 "screenshot": "0769779032616734_21.png",319 "action": "CLICK",320 "info": [321 [322 944,323 502324 ],325 [326 944,327 502328 ]329 ],330 "ps": "[(944, 502)]"331 },332 {333 "step": 22,334 "screenshot": "0769779032616734_22.png",335 "action": "CLICK",336 "info": "KEY_HOME",337 "ps": ""338 },339 {340 "step": 23,341 "screenshot": "0769779032616734_23.png",342 "action": "CLICK",343 "info": [344 [345 454,346 505347 ],348 [349 454,350 505351 ]352 ],353 "ps": "[(454, 505)]"354 },355 {356 "step": 24,357 "screenshot": "0769779032616734_24.png",358 "action": "CLICK",359 "info": [360 [361 777,362 842363 ],364 [365 777,366 842367 ]368 ],369 "ps": "[(777, 842)]"370 },371 {372 "step": 25,373 "screenshot": "0769779032616734_25.png",374 "action": "CLICK",375 "info": [376 [377 594,378 589379 ],380 [381 594,382 589383 ]384 ],385 "ps": "[(594, 589)]"386 },387 {388 "step": 26,389 "screenshot": "0769779032616734_26.png",390 "action": "CLICK",391 "info": [392 [393 557,394 478395 ],396 [397 557,398 478399 ]400 ],401 "ps": "[(557, 478)]"402 },403 {404 "step": 27,405 "screenshot": "0769779032616734_27.png",406 "action": "CLICK",407 "info": [408 [409 804,410 730411 ],412 [413 804,414 730415 ]416 ],417 "ps": "[(804, 730)]"418 },419 {420 "step": 28,421 "screenshot": "0769779032616734_28.png",422 "action": "CLICK",423 "info": [424 [425 626,426 409427 ],428 [429 626,430 409431 ]432 ],433 "ps": "[(626, 409)]"434 },435 {436 "step": 29,437 "screenshot": "0769779032616734_29.png",438 "action": "COMPLETE",439 "info": "",440 "ps": "liudehu19294094"441 }442 ]443}