OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0759690525878634",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Clock",15 "Chrome",16 "Messenger",17 "Ebay"18 ],19 "meta_task": "Plan a movie night by selecting a {} movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",20 "task": "Plan a movie night by selecting a mystery movie, add some snacks to the cart on a shopping app, send invitations through a messaging app, and set a clock to remind yourself.",21 "instruction": "Organize a movie night by first choosing a mystery movie using Chrome. Next, add some snacks to your cart on Ebay. Then, send out invitations to Victor James via Messenger. Finally, set a reminder on Clock to not forget about the event."22 },23 "step_length": 30,24 "steps": [25 {26 "step": 0,27 "screenshot": "0759690525878634_0.png",28 "action": "CLICK",29 "info": [30 [31 174,32 63433 ],34 [35 174,36 63437 ]38 ],39 "ps": "[(174, 634)]"40 },41 {42 "step": 1,43 "screenshot": "0759690525878634_1.png",44 "action": "CLICK",45 "info": [46 [47 409,48 27849 ],50 [51 409,52 27853 ]54 ],55 "ps": "[(409, 278)]"56 },57 {58 "step": 2,59 "screenshot": "0759690525878634_2.png",60 "action": "TEXT",61 "info": "mystery MOVIES",62 "ps": ""63 },64 {65 "step": 3,66 "screenshot": "0759690525878634_3.png",67 "action": "CLICK",68 "info": [69 [70 936,71 91872 ],73 [74 936,75 91876 ]77 ],78 "ps": "[(936, 918)]"79 },80 {81 "step": 4,82 "screenshot": "0759690525878634_4.png",83 "action": "CLICK",84 "info": [85 [86 140,87 39488 ],89 [90 140,91 39492 ]93 ],94 "ps": "[(140, 394)]"95 },96 {97 "step": 5,98 "screenshot": "0759690525878634_5.png",99 "action": "CLICK",100 "info": [101 [102 152,103 384104 ],105 [106 152,107 384108 ]109 ],110 "ps": "[(152, 384)]"111 },112 {113 "step": 6,114 "screenshot": "0759690525878634_6.png",115 "action": "CLICK",116 "info": "KEY_HOME",117 "ps": ""118 },119 {120 "step": 7,121 "screenshot": "0759690525878634_7.png",122 "action": "CLICK",123 "info": [124 [125 352,126 365127 ],128 [129 352,130 365131 ]132 ],133 "ps": "[(352, 365)]"134 },135 {136 "step": 8,137 "screenshot": "0759690525878634_8.png",138 "action": "CLICK",139 "info": [140 [141 304,142 141143 ],144 [145 304,146 141147 ]148 ],149 "ps": "[(304, 141)]"150 },151 {152 "step": 9,153 "screenshot": "0759690525878634_9.png",154 "action": "TEXT",155 "info": "snacks",156 "ps": ""157 },158 {159 "step": 10,160 "screenshot": "0759690525878634_10.png",161 "action": "CLICK",162 "info": [163 [164 925,165 914166 ],167 [168 925,169 914170 ]171 ],172 "ps": "[(925, 914)]"173 },174 {175 "step": 11,176 "screenshot": "0759690525878634_11.png",177 "action": "CLICK",178 "info": [179 [180 694,181 444182 ],183 [184 694,185 444186 ]187 ],188 "ps": "[(694, 444)]"189 },190 {191 "step": 12,192 "screenshot": "0759690525878634_12.png",193 "action": "SCROLL",194 "info": [195 [196 467,197 899198 ],199 [200 562,201 410202 ]203 ],204 "ps": "[(467, 899), (467, 898), (467, 896), (467, 893), (467, 887), (467, 881), (462, 879), (462, 877), (462, 874), (462, 873), (462, 871), (462, 867), (462, 862), (462, 858), (462, 851), (462, 846), (462, 839), (462, 835), (462, 830), (462, 826), (462, 822), (462, 817), (462, 813), (462, 811), (462, 807), (462, 803), (462, 801), (465, 797), (465, 792), (465, 788), (467, 787), (467, 782), (467, 778), (467, 777), (467, 774), (467, 771), (467, 769), (467, 766), (474, 761), (474, 759), (477, 756), (477, 754), (479, 750), (479, 748), (479, 746), (479, 743), (479, 740), (479, 737), (479, 735), (479, 733), (479, 729), (479, 727), (479, 724), (479, 719), (479, 715), (479, 713), (479, 711), (479, 705), (479, 703), (479, 699), (479, 698), (479, 696), (479, 691), (479, 687), (479, 682), (479, 681), (479, 678), (479, 676), (479, 674), (479, 669), (479, 663), (479, 656), (479, 652), (479, 647), (479, 643), (479, 637), (479, 633), (479, 629), (479, 624), (485, 618), (485, 613), (485, 609), (485, 604), (487, 600), (487, 597), (487, 596), (489, 592), (489, 588), (489, 582), (489, 579), (497, 575), (497, 572), (497, 569), (497, 566), (497, 562), (497, 559), (497, 555), (497, 551), (504, 547), (504, 543), (506, 539), (506, 534), (512, 528), (512, 523), (516, 519), (521, 514), (521, 510), (521, 506), (521, 502), (521, 499), (521, 498), (524, 494), (524, 489), (524, 487), (531, 482), (531, 477), (533, 476), (533, 474), (533, 472), (533, 469), (533, 467), (536, 465), (536, 462), (536, 460), (543, 455), (543, 452), (543, 448), (543, 446), (550, 441), (550, 440), (550, 438), (550, 434), (555, 434), (555, 430), (555, 428), (555, 427), (555, 426), (555, 424), (555, 422), (555, 418), (562, 415), (562, 413), (562, 410)]"205 },206 {207 "step": 13,208 "screenshot": "0759690525878634_13.png",209 "action": "CLICK",210 "info": [211 [212 836,213 643214 ],215 [216 836,217 643218 ]219 ],220 "ps": "[(836, 643)]"221 },222 {223 "step": 14,224 "screenshot": "0759690525878634_14.png",225 "action": "CLICK",226 "info": [227 [228 697,229 840230 ],231 [232 697,233 840234 ]235 ],236 "ps": "[(697, 840)]"237 },238 {239 "step": 15,240 "screenshot": "0759690525878634_15.png",241 "action": "CLICK",242 "info": "KEY_HOME",243 "ps": ""244 },245 {246 "step": 16,247 "screenshot": "0759690525878634_16.png",248 "action": "CLICK",249 "info": [250 [251 819,252 110253 ],254 [255 819,256 110257 ]258 ],259 "ps": "[(819, 110)]"260 },261 {262 "step": 17,263 "screenshot": "0759690525878634_17.png",264 "action": "CLICK",265 "info": [266 [267 358,268 330269 ],270 [271 358,272 330273 ]274 ],275 "ps": "[(358, 330)]"276 },277 {278 "step": 18,279 "screenshot": "0759690525878634_18.png",280 "action": "CLICK",281 "info": [282 [283 543,284 925285 ],286 [287 543,288 925289 ]290 ],291 "ps": "[(543, 925)]"292 },293 {294 "step": 19,295 "screenshot": "0759690525878634_19.png",296 "action": "TEXT",297 "info": "Do you want to watch a movie this Sunday night?",298 "ps": ""299 },300 {301 "step": 20,302 "screenshot": "0759690525878634_20.png",303 "action": "CLICK",304 "info": [305 [306 940,307 607308 ],309 [310 940,311 607312 ]313 ],314 "ps": "[(940, 607)]"315 },316 {317 "step": 21,318 "screenshot": "0759690525878634_21.png",319 "action": "CLICK",320 "info": "KEY_HOME",321 "ps": ""322 },323 {324 "step": 22,325 "screenshot": "0759690525878634_22.png",326 "action": "CLICK",327 "info": [328 [329 843,330 500331 ],332 [333 843,334 500335 ]336 ],337 "ps": "[(843, 500)]"338 },339 {340 "step": 23,341 "screenshot": "0759690525878634_23.png",342 "action": "CLICK",343 "info": [344 [345 482,346 806347 ],348 [349 482,350 806351 ]352 ],353 "ps": "[(482, 806)]"354 },355 {356 "step": 24,357 "screenshot": "0759690525878634_24.png",358 "action": "CLICK",359 "info": [360 [361 762,362 361363 ],364 [365 762,366 361367 ]368 ],369 "ps": "[(762, 361)]"370 },371 {372 "step": 25,373 "screenshot": "0759690525878634_25.png",374 "action": "CLICK",375 "info": [376 [377 287,378 607379 ],380 [381 287,382 607383 ]384 ],385 "ps": "[(287, 607)]"386 },387 {388 "step": 26,389 "screenshot": "0759690525878634_26.png",390 "action": "CLICK",391 "info": [392 [393 397,394 653395 ],396 [397 397,398 653399 ]400 ],401 "ps": "[(397, 653)]"402 },403 {404 "step": 27,405 "screenshot": "0759690525878634_27.png",406 "action": "CLICK",407 "info": [408 [409 812,410 745411 ],412 [413 812,414 745415 ]416 ],417 "ps": "[(812, 745)]"418 },419 {420 "step": 28,421 "screenshot": "0759690525878634_28.png",422 "action": "CLICK",423 "info": [424 [425 143,426 328427 ],428 [429 143,430 328431 ]432 ],433 "ps": "[(143, 328)]"434 },435 {436 "step": 29,437 "screenshot": "0759690525878634_29.png",438 "action": "COMPLETE",439 "info": "",440 "ps": "Victor James"441 }442 ]443}