OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0474923361897823",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1280,8 "w": 720,9 "device_name": "Small Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Bing: chat with AI & GPT4",15 "Calendar",16 "Ebay"17 ],18 "meta_task": "Look up a/an {} class, buy the thing needed in the class, and set a calendar reminder to study.",19 "task": "Look up a/an Spanish language class, buy the thing needed in the class, and set a calendar reminder to study.",20 "instruction": "Use Bing: chat with AI & GPT4 to look up a Spanish language class, then purchase the necessary materials for the class on Ebay, and finally set a reminder to study in your Calendar."21 },22 "step_length": 22,23 "steps": [24 {25 "step": 0,26 "screenshot": "0474923361897823_0.png",27 "action": "CLICK",28 "info": [29 [30 137,31 23032 ],33 [34 137,35 23036 ]37 ],38 "ps": "[(137, 230)]"39 },40 {41 "step": 1,42 "screenshot": "0474923361897823_1.png",43 "action": "CLICK",44 "info": [45 [46 293,47 43248 ],49 [50 293,51 43252 ]53 ],54 "ps": "[(293, 432)]"55 },56 {57 "step": 2,58 "screenshot": "0474923361897823_2.png",59 "action": "TEXT",60 "info": "spanish language class",61 "ps": ""62 },63 {64 "step": 3,65 "screenshot": "0474923361897823_3.png",66 "action": "CLICK",67 "info": [68 [69 388,70 16471 ],72 [73 388,74 16475 ]76 ],77 "ps": "[(388, 164)]"78 },79 {80 "step": 4,81 "screenshot": "0474923361897823_4.png",82 "action": "SCROLL",83 "info": [84 [85 640,86 61087 ],88 [89 583,90 20891 ]92 ],93 "ps": "[(640, 610), (640, 607), (640, 606), (640, 603), (640, 597), (640, 590), (640, 585), (640, 580), (640, 576), (637, 562), (633, 553), (633, 546), (631, 535), (627, 522), (626, 510), (626, 502), (622, 486), (622, 478), (619, 471), (619, 458), (615, 442), (613, 428), (609, 416), (608, 406), (604, 392), (601, 381), (597, 367), (595, 356), (595, 349), (591, 341), (591, 332), (591, 325), (586, 317), (586, 307), (586, 294), (586, 287), (586, 281), (586, 274), (586, 261), (586, 253), (586, 250), (586, 243), (586, 240), (586, 237), (586, 235), (586, 232), (586, 230), (586, 227), (586, 224), (586, 221), (586, 220), (586, 217), (583, 217), (583, 214), (583, 214), (583, 211), (583, 210), (583, 208)]"94 },95 {96 "step": 5,97 "screenshot": "0474923361897823_5.png",98 "action": "CLICK",99 "info": [100 [101 358,102 556103 ],104 [105 358,106 556107 ]108 ],109 "ps": "[(358, 556)]"110 },111 {112 "step": 6,113 "screenshot": "0474923361897823_6.png",114 "action": "CLICK",115 "info": "KEY_HOME",116 "ps": ""117 },118 {119 "step": 7,120 "screenshot": "0474923361897823_7.png",121 "action": "CLICK",122 "info": [123 [124 865,125 82126 ],127 [128 865,129 82130 ]131 ],132 "ps": "[(865, 82)]"133 },134 {135 "step": 8,136 "screenshot": "0474923361897823_8.png",137 "action": "CLICK",138 "info": [139 [140 352,141 183142 ],143 [144 352,145 183146 ]147 ],148 "ps": "[(352, 183)]"149 },150 {151 "step": 9,152 "screenshot": "0474923361897823_9.png",153 "action": "TEXT",154 "info": "spanish learning book",155 "ps": ""156 },157 {158 "step": 10,159 "screenshot": "0474923361897823_10.png",160 "action": "CLICK",161 "info": [162 [163 918,164 885165 ],166 [167 918,168 885169 ]170 ],171 "ps": "[(918, 885)]"172 },173 {174 "step": 11,175 "screenshot": "0474923361897823_11.png",176 "action": "CLICK",177 "info": [178 [179 693,180 413181 ],182 [183 693,184 413185 ]186 ],187 "ps": "[(693, 413)]"188 },189 {190 "step": 12,191 "screenshot": "0474923361897823_12.png",192 "action": "SCROLL",193 "info": [194 [195 412,196 717197 ],198 [199 461,200 289201 ]202 ],203 "ps": "[(412, 717), (412, 717), (412, 714), (412, 708), (412, 703), (412, 701), (412, 696), (412, 693), (412, 690), (412, 683), (412, 676), (412, 667), (412, 660), (412, 647), (412, 634), (412, 621), (412, 610), (412, 603), (416, 592), (418, 577), (422, 569), (424, 559), (429, 546), (429, 540), (430, 528), (434, 515), (436, 503), (436, 499), (440, 489), (440, 482), (443, 466), (445, 462), (445, 455), (448, 448), (448, 442), (454, 428), (454, 424), (454, 418), (454, 405), (458, 398), (458, 392), (458, 385), (461, 374), (461, 367), (461, 361), (461, 352), (461, 346), (461, 342), (461, 338), (461, 332), (461, 328), (461, 324), (461, 321), (461, 318), (461, 315), (461, 310), (461, 308), (461, 307), (461, 304), (461, 300), (461, 298), (461, 297), (461, 295), (461, 294), (461, 292), (461, 290), (461, 289)]"204 },205 {206 "step": 13,207 "screenshot": "0474923361897823_13.png",208 "action": "SCROLL",209 "info": [210 [211 544,212 735213 ],214 [215 513,216 285217 ]218 ],219 "ps": "[(544, 735), (544, 733), (541, 730), (541, 728), (537, 727), (537, 724), (537, 720), (536, 711), (531, 708), (530, 700), (526, 688), (520, 673), (518, 663), (513, 651), (512, 637), (508, 623), (506, 616), (502, 603), (499, 585), (495, 572), (494, 557), (490, 543), (488, 532), (488, 520), (488, 509), (488, 495), (488, 485), (488, 478), (488, 465), (488, 458), (488, 448), (488, 442), (488, 432), (488, 418), (488, 409), (490, 399), (494, 388), (494, 381), (495, 374), (495, 362), (495, 354), (499, 349), (499, 341), (499, 334), (502, 328), (502, 322), (502, 317), (502, 312), (502, 308), (502, 305), (506, 300), (506, 297), (508, 297), (508, 295), (508, 294), (508, 292), (508, 290), (508, 289), (512, 289), (512, 287), (513, 285)]"220 },221 {222 "step": 14,223 "screenshot": "0474923361897823_14.png",224 "action": "CLICK",225 "info": [226 [227 424,228 688229 ],230 [231 424,232 688233 ]234 ],235 "ps": "[(424, 688)]"236 },237 {238 "step": 15,239 "screenshot": "0474923361897823_15.png",240 "action": "CLICK",241 "info": "KEY_HOME",242 "ps": ""243 },244 {245 "step": 16,246 "screenshot": "0474923361897823_16.png",247 "action": "CLICK",248 "info": [249 [250 865,251 351252 ],253 [254 865,255 351256 ]257 ],258 "ps": "[(865, 351)]"259 },260 {261 "step": 17,262 "screenshot": "0474923361897823_17.png",263 "action": "CLICK",264 "info": [265 [266 872,267 888268 ],269 [270 872,271 888272 ]273 ],274 "ps": "[(872, 888)]"275 },276 {277 "step": 18,278 "screenshot": "0474923361897823_18.png",279 "action": "CLICK",280 "info": [281 [282 872,283 888284 ],285 [286 872,287 888288 ]289 ],290 "ps": "[(872, 888)]"291 },292 {293 "step": 19,294 "screenshot": "0474923361897823_19.png",295 "action": "TEXT",296 "info": "spanish language class",297 "ps": ""298 },299 {300 "step": 20,301 "screenshot": "0474923361897823_20.png",302 "action": "CLICK",303 "info": [304 [305 861,306 99307 ],308 [309 861,310 99311 ]312 ],313 "ps": "[(861, 99)]"314 },315 {316 "step": 21,317 "screenshot": "0474923361897823_21.png",318 "action": "COMPLETE",319 "info": "",320 "ps": ""321 }322 ]323}