OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "1221334750816089",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Chrome",15 "Daily Forecast",16 "Gmail"17 ],18 "meta_task": "Look for a new hiking trail, check the weather forecast for the weekend and invite friends to join.",19 "task": "Look for a new hiking trail, check the weather forecast for the weekend and invite friends to join.",20 "instruction": "Utilize Chrome to search for a new hiking trail, then use Daily Forecast to check the weekend weather. Finally, invite caba62244@gmail.com to join using Gmail."21 },22 "step_length": 30,23 "steps": [24 {25 "step": 0,26 "screenshot": "1221334750816089_0.png",27 "action": "CLICK",28 "info": [29 [30 164,31 66032 ],33 [34 164,35 66036 ]37 ],38 "ps": "[(164, 660)]"39 },40 {41 "step": 1,42 "screenshot": "1221334750816089_1.png",43 "action": "CLICK",44 "info": [45 [46 311,47 27348 ],49 [50 311,51 27352 ]53 ],54 "ps": "[(311, 273)]"55 },56 {57 "step": 2,58 "screenshot": "1221334750816089_2.png",59 "action": "TEXT",60 "info": "hiking trail",61 "ps": ""62 },63 {64 "step": 3,65 "screenshot": "1221334750816089_3.png",66 "action": "CLICK",67 "info": [68 [69 946,70 91171 ],72 [73 946,74 91175 ]76 ],77 "ps": "[(946, 911)]"78 },79 {80 "step": 4,81 "screenshot": "1221334750816089_4.png",82 "action": "SCROLL",83 "info": [84 [85 512,86 77487 ],88 [89 391,90 18291 ]92 ],93 "ps": "[(512, 774), (510, 773), (510, 769), (505, 765), (505, 759), (498, 754), (498, 752), (498, 748), (498, 743), (498, 739), (498, 737), (498, 736), (498, 732), (498, 728), (498, 724), (498, 719), (498, 715), (493, 710), (493, 708), (493, 705), (493, 701), (493, 695), (493, 691), (493, 687), (493, 680), (493, 672), (493, 666), (493, 659), (493, 651), (493, 642), (493, 632), (493, 613), (493, 604), (493, 593), (493, 584), (486, 580), (486, 577), (486, 569), (478, 565), (478, 564), (478, 561), (478, 555), (471, 549), (471, 545), (469, 541), (462, 537), (462, 532), (459, 528), (459, 521), (459, 517), (459, 516), (459, 512), (451, 507), (451, 505), (451, 500), (451, 494), (451, 484), (451, 467), (451, 459), (451, 450), (451, 439), (451, 432), (451, 423), (444, 418), (444, 414), (444, 407), (444, 401), (444, 395), (444, 387), (444, 379), (439, 365), (439, 353), (439, 347), (439, 341), (439, 333), (439, 327), (439, 321), (439, 312), (439, 303), (435, 301), (435, 297), (435, 292), (430, 287), (427, 283), (427, 275), (427, 268), (427, 264), (427, 259), (427, 255), (427, 251), (427, 247), (427, 242), (427, 238), (420, 233), (420, 231), (417, 226), (417, 222), (417, 217), (411, 215), (411, 214), (411, 210), (411, 208), (411, 206), (403, 203), (398, 199), (398, 198), (398, 197), (398, 196), (398, 194), (391, 191), (391, 189), (391, 187), (391, 184), (391, 183), (391, 182)]"94 },95 {96 "step": 5,97 "screenshot": "1221334750816089_5.png",98 "action": "SCROLL",99 "info": [100 [101 439,102 883103 ],104 [105 378,106 263107 ]108 ],109 "ps": "[(439, 883), (435, 883), (430, 883), (424, 881), (424, 879), (423, 870), (412, 862), (405, 856), (405, 852), (398, 839), (388, 824), (384, 822), (384, 819), (384, 814), (384, 810), (384, 804), (384, 798), (384, 794), (384, 789), (384, 783), (384, 773), (384, 766), (384, 762), (384, 757), (384, 748), (384, 742), (384, 738), (384, 729), (384, 718), (384, 709), (384, 701), (384, 689), (384, 679), (384, 670), (384, 659), (384, 650), (384, 639), (384, 634), (384, 610), (384, 582), (384, 566), (384, 555), (384, 544), (384, 535), (384, 527), (384, 519), (384, 508), (384, 499), (384, 493), (384, 487), (384, 458), (384, 444), (384, 436), (384, 429), (384, 425), (384, 420), (384, 416), (384, 414), (384, 409), (384, 406), (384, 404), (384, 402), (384, 399), (384, 397), (384, 392), (384, 389), (378, 384), (378, 380), (378, 378), (378, 376), (378, 371), (378, 368), (378, 367), (378, 365), (378, 360), (378, 357), (378, 353), (378, 350), (378, 344), (378, 339), (378, 332), (378, 324), (378, 315), (378, 309), (378, 303), (378, 294), (378, 289), (378, 283), (378, 278), (378, 274), (378, 272), (378, 268), (378, 266), (378, 263)]"110 },111 {112 "step": 6,113 "screenshot": "1221334750816089_6.png",114 "action": "CLICK",115 "info": [116 [117 435,118 755119 ],120 [121 435,122 755123 ]124 ],125 "ps": "[(435, 755)]"126 },127 {128 "step": 7,129 "screenshot": "1221334750816089_7.png",130 "action": "SCROLL",131 "info": [132 [133 423,134 682135 ],136 [137 378,138 267139 ]140 ],141 "ps": "[(423, 682), (417, 682), (417, 680), (415, 679), (415, 677), (412, 675), (412, 672), (412, 670), (412, 668), (405, 664), (408, 661), (408, 659), (405, 656), (403, 654), (403, 649), (403, 647), (396, 644), (396, 640), (396, 636), (396, 632), (396, 629), (396, 627), (396, 626), (396, 623), (396, 619), (396, 614), (396, 613), (396, 611), (396, 609), (396, 607), (396, 604), (396, 603), (396, 602), (396, 600), (396, 599), (396, 596), (396, 593), (396, 591), (396, 588), (396, 586), (396, 582), (396, 580), (396, 576), (396, 575), (396, 572), (396, 572), (396, 570), (396, 567), (396, 565), (396, 563), (396, 559), (396, 557), (396, 553), (396, 549), (396, 545), (396, 543), (396, 539), (396, 537), (396, 533), (396, 529), (396, 524), (396, 522), (396, 521), (396, 518), (396, 514), (396, 509), (396, 507), (396, 505), (396, 500), (396, 496), (396, 493), (396, 488), (396, 484), (396, 482), (396, 479), (396, 474), (396, 470), (396, 465), (396, 463), (396, 454), (396, 447), (396, 440), (396, 435), (396, 432), (396, 429), (396, 428), (396, 424), (396, 419), (396, 418), (396, 414), (396, 409), (396, 405), (396, 403), (396, 402), (396, 397), (396, 393), (396, 391), (396, 389), (396, 384), (396, 382), (396, 380), (396, 376), (396, 371), (396, 367), (396, 362), (396, 358), (396, 354), (396, 352), (396, 347), (396, 346), (396, 343), (396, 342), (396, 339), (396, 335), (396, 331), (396, 329), (396, 325), (396, 320), (396, 313), (396, 311), (391, 311), (391, 308), (391, 306), (391, 302), (391, 300), (391, 296), (391, 292), (391, 289), (391, 286), (384, 284), (384, 280), (384, 279), (381, 278), (378, 276), (378, 273), (378, 271), (378, 267)]"142 },143 {144 "step": 8,145 "screenshot": "1221334750816089_8.png",146 "action": "CLICK",147 "info": "KEY_HOME",148 "ps": ""149 },150 {151 "step": 9,152 "screenshot": "1221334750816089_9.png",153 "action": "CLICK",154 "info": [155 [156 136,157 508158 ],159 [160 136,161 508162 ]163 ],164 "ps": "[(136, 508)]"165 },166 {167 "step": 10,168 "screenshot": "1221334750816089_10.png",169 "action": "CLICK",170 "info": [171 [172 649,173 57174 ],175 [176 649,177 57178 ]179 ],180 "ps": "[(649, 57)]"181 },182 {183 "step": 11,184 "screenshot": "1221334750816089_11.png",185 "action": "CLICK",186 "info": [187 [188 82,189 63190 ],191 [192 82,193 63194 ]195 ],196 "ps": "[(82, 63)]"197 },198 {199 "step": 12,200 "screenshot": "1221334750816089_12.png",201 "action": "CLICK",202 "info": [203 [204 907,205 59206 ],207 [208 907,209 59210 ]211 ],212 "ps": "[(907, 59)]"213 },214 {215 "step": 13,216 "screenshot": "1221334750816089_13.png",217 "action": "CLICK",218 "info": [219 [220 52,221 30222 ],223 [224 52,225 30226 ]227 ],228 "ps": "[(52, 30)]"229 },230 {231 "step": 14,232 "screenshot": "1221334750816089_14.png",233 "action": "CLICK",234 "info": [235 [236 74,237 26238 ],239 [240 74,241 26242 ]243 ],244 "ps": "[(74, 26)]"245 },246 {247 "step": 15,248 "screenshot": "1221334750816089_15.png",249 "action": "CLICK",250 "info": [251 [252 64,253 28254 ],255 [256 64,257 28258 ]259 ],260 "ps": "[(64, 28)]"261 },262 {263 "step": 16,264 "screenshot": "1221334750816089_16.png",265 "action": "CLICK",266 "info": [267 [268 60,269 23270 ],271 [272 60,273 23274 ]275 ],276 "ps": "[(60, 23)]"277 },278 {279 "step": 17,280 "screenshot": "1221334750816089_17.png",281 "action": "CLICK",282 "info": [283 [284 312,285 55286 ],287 [288 312,289 55290 ]291 ],292 "ps": "[(312, 55)]"293 },294 {295 "step": 18,296 "screenshot": "1221334750816089_18.png",297 "action": "TEXT",298 "info": "California",299 "ps": ""300 },301 {302 "step": 19,303 "screenshot": "1221334750816089_19.png",304 "action": "CLICK",305 "info": [306 [307 500,308 302309 ],310 [311 500,312 302313 ]314 ],315 "ps": "[(500, 302)]"316 },317 {318 "step": 20,319 "screenshot": "1221334750816089_20.png",320 "action": "CLICK",321 "info": [322 [323 741,324 259325 ],326 [327 741,328 259329 ]330 ],331 "ps": "[(741, 259)]"332 },333 {334 "step": 21,335 "screenshot": "1221334750816089_21.png",336 "action": "CLICK",337 "info": "KEY_HOME",338 "ps": ""339 },340 {341 "step": 22,342 "screenshot": "1221334750816089_22.png",343 "action": "CLICK",344 "info": [345 [346 388,347 112348 ],349 [350 388,351 112352 ]353 ],354 "ps": "[(388, 112)]"355 },356 {357 "step": 23,358 "screenshot": "1221334750816089_23.png",359 "action": "CLICK",360 "info": [361 [362 719,363 878364 ],365 [366 719,367 878368 ]369 ],370 "ps": "[(719, 878)]"371 },372 {373 "step": 24,374 "screenshot": "1221334750816089_24.png",375 "action": "TEXT",376 "info": "caba62244@gmail.com",377 "ps": ""378 },379 {380 "step": 25,381 "screenshot": "1221334750816089_25.png",382 "action": "CLICK",383 "info": [384 [385 136,386 257387 ],388 [389 136,390 257391 ]392 ],393 "ps": "[(136, 257)]"394 },395 {396 "step": 26,397 "screenshot": "1221334750816089_26.png",398 "action": "CLICK",399 "info": [400 [401 162,402 308403 ],404 [405 162,406 308407 ]408 ],409 "ps": "[(162, 308)]"410 },411 {412 "step": 27,413 "screenshot": "1221334750816089_27.png",414 "action": "TEXT",415 "info": "Do you want to go to California with me for a hiking trail next Sunday?",416 "ps": ""417 },418 {419 "step": 28,420 "screenshot": "1221334750816089_28.png",421 "action": "CLICK",422 "info": [423 [424 836,425 61426 ],427 [428 836,429 61430 ]431 ],432 "ps": "[(836, 61)]"433 },434 {435 "step": 29,436 "screenshot": "1221334750816089_29.png",437 "action": "COMPLETE",438 "info": "",439 "ps": "caba62244@gmail.com"440 }441 ]442}