OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0334057142974503",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1280,8 "w": 720,9 "device_name": "Small Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Amazon",15 "Calendar",16 "DuckDuckGo"17 ],18 "meta_task": "Look up a/an {} class, buy the thing needed in the class, and set a calendar reminder to study.",19 "task": "Look up a/an Chinese language class, buy the thing needed in the class, and set a calendar reminder to study.",20 "instruction": "Search for a Chinese language class using DuckDuckGo, purchase the materials required for the class on Amazon, and create a study reminder in your Calendar app."21 },22 "step_length": 21,23 "steps": [24 {25 "step": 0,26 "screenshot": "0334057142974503_0.png",27 "action": "CLICK",28 "info": [29 [30 673,31 21032 ],33 [34 674,35 21036 ]37 ],38 "ps": "[(673, 210), (674, 210)]"39 },40 {41 "step": 1,42 "screenshot": "0334057142974503_1.png",43 "action": "TEXT",44 "info": "chinese language class",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "0334057142974503_2.png",50 "action": "CLICK",51 "info": [52 [53 926,54 88155 ],56 [57 926,58 88159 ]60 ],61 "ps": "[(926, 881)]"62 },63 {64 "step": 3,65 "screenshot": "0334057142974503_3.png",66 "action": "CLICK",67 "info": [68 [69 334,70 69671 ],72 [73 334,74 69675 ]76 ],77 "ps": "[(334, 696)]"78 },79 {80 "step": 4,81 "screenshot": "0334057142974503_4.png",82 "action": "CLICK",83 "info": "KEY_HOME",84 "ps": ""85 },86 {87 "step": 5,88 "screenshot": "0334057142974503_5.png",89 "action": "CLICK",90 "info": [91 [92 484,93 7094 ],95 [96 484,97 7098 ]99 ],100 "ps": "[(484, 70)]"101 },102 {103 "step": 6,104 "screenshot": "0334057142974503_6.png",105 "action": "CLICK",106 "info": [107 [108 388,109 89110 ],111 [112 388,113 89114 ]115 ],116 "ps": "[(388, 89)]"117 },118 {119 "step": 7,120 "screenshot": "0334057142974503_7.png",121 "action": "TEXT",122 "info": "chinese learning book",123 "ps": ""124 },125 {126 "step": 8,127 "screenshot": "0334057142974503_8.png",128 "action": "CLICK",129 "info": [130 [131 890,132 881133 ],134 [135 890,136 881137 ]138 ],139 "ps": "[(890, 881)]"140 },141 {142 "step": 9,143 "screenshot": "0334057142974503_9.png",144 "action": "CLICK",145 "info": [146 [147 674,148 651149 ],150 [151 674,152 651153 ]154 ],155 "ps": "[(674, 651)]"156 },157 {158 "step": 10,159 "screenshot": "0334057142974503_10.png",160 "action": "SCROLL",161 "info": [162 [163 555,164 774165 ],166 [167 541,168 338169 ]170 ],171 "ps": "[(555, 774), (555, 771), (554, 767), (549, 761), (548, 758), (544, 755), (544, 750), (541, 743), (537, 740), (536, 732), (531, 727), (531, 720), (530, 708), (526, 698), (523, 686), (520, 678), (518, 667), (518, 654), (513, 642), (513, 633), (513, 614), (513, 599), (513, 585), (513, 574), (513, 553), (513, 539), (513, 528), (513, 507), (513, 499), (513, 475), (513, 468), (513, 455), (518, 445), (520, 435), (520, 428), (523, 421), (523, 418), (523, 414), (526, 411), (526, 406), (526, 403), (526, 398), (526, 391), (530, 388), (530, 385), (530, 382), (530, 379), (531, 374), (531, 371), (531, 369), (531, 367), (531, 364), (531, 362), (531, 361), (536, 357), (536, 356), (536, 352), (536, 351), (537, 347), (537, 346), (537, 344), (537, 342), (537, 341), (537, 339), (537, 338), (541, 338)]"172 },173 {174 "step": 11,175 "screenshot": "0334057142974503_11.png",176 "action": "SCROLL",177 "info": [178 [179 568,180 653181 ],182 [183 548,184 253185 ]186 ],187 "ps": "[(568, 653), (568, 649), (566, 646), (562, 642), (562, 639), (555, 633), (555, 629), (554, 624), (549, 619), (549, 614), (549, 606), (548, 600), (544, 590), (544, 584), (541, 576), (541, 569), (541, 560), (541, 553), (541, 549), (541, 542), (541, 535), (541, 528), (541, 517), (541, 503), (541, 495), (541, 485), (541, 476), (541, 471), (541, 463), (541, 453), (541, 446), (544, 438), (544, 432), (548, 428), (548, 419), (548, 413), (548, 399), (548, 391), (548, 385), (548, 378), (544, 367), (544, 357), (544, 351), (544, 344), (544, 335), (544, 324), (544, 317), (544, 310), (544, 304), (544, 298), (544, 294), (544, 290), (544, 287), (544, 282), (544, 281), (544, 277), (544, 274), (544, 271), (544, 271), (544, 268), (544, 267), (544, 265), (544, 264), (544, 260), (548, 258), (548, 257), (548, 255), (548, 253)]"188 },189 {190 "step": 12,191 "screenshot": "0334057142974503_12.png",192 "action": "SCROLL",193 "info": [194 [195 615,196 817197 ],198 [199 604,200 347201 ]202 ],203 "ps": "[(615, 817), (615, 814), (615, 810), (615, 807), (615, 802), (609, 794), (609, 790), (609, 781), (609, 764), (604, 750), (604, 733), (604, 724), (604, 703), (597, 690), (597, 670), (597, 653), (597, 641), (597, 626), (597, 614), (597, 596), (597, 582), (597, 564), (597, 553), (597, 539), (597, 528), (597, 509), (597, 493), (597, 478), (597, 468), (597, 460), (597, 439), (601, 432), (601, 419), (601, 409), (601, 403), (601, 395), (601, 388), (601, 381), (601, 378), (601, 374), (601, 371), (604, 365), (604, 364), (604, 362), (604, 361), (604, 359), (604, 357), (604, 356), (604, 354), (604, 352), (604, 351), (604, 349), (604, 347)]"204 },205 {206 "step": 13,207 "screenshot": "0334057142974503_13.png",208 "action": "CLICK",209 "info": [210 [211 508,212 851213 ],214 [215 508,216 851217 ]218 ],219 "ps": "[(508, 851)]"220 },221 {222 "step": 14,223 "screenshot": "0334057142974503_14.png",224 "action": "CLICK",225 "info": "KEY_HOME",226 "ps": ""227 },228 {229 "step": 15,230 "screenshot": "0334057142974503_15.png",231 "action": "CLICK",232 "info": [233 [234 861,235 351236 ],237 [238 861,239 351240 ]241 ],242 "ps": "[(861, 351)]"243 },244 {245 "step": 16,246 "screenshot": "0334057142974503_16.png",247 "action": "CLICK",248 "info": [249 [250 872,251 885252 ],253 [254 872,255 885256 ]257 ],258 "ps": "[(872, 885)]"259 },260 {261 "step": 17,262 "screenshot": "0334057142974503_17.png",263 "action": "CLICK",264 "info": [265 [266 872,267 885268 ],269 [270 872,271 885272 ]273 ],274 "ps": "[(872, 885)]"275 },276 {277 "step": 18,278 "screenshot": "0334057142974503_18.png",279 "action": "TEXT",280 "info": "chinese language class",281 "ps": ""282 },283 {284 "step": 19,285 "screenshot": "0334057142974503_19.png",286 "action": "CLICK",287 "info": [288 [289 884,290 87291 ],292 [293 884,294 87295 ]296 ],297 "ps": "[(884, 87)]"298 },299 {300 "step": 20,301 "screenshot": "0334057142974503_20.png",302 "action": "COMPLETE",303 "info": "",304 "ps": ""305 }306 ]307}