OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0055180990380237",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1280,8 "w": 720,9 "device_name": "Small Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Google Keep",15 "The Weather Channel",16 "Traveloka",17 "Tripadvisor"18 ],19 "meta_task": "Search for a travel guide to visit {}, record the resource website, then select a day without rain, finally book a flight from {}.",20 "task": "Search for a travel guide to visit Bali Island, record the resource website, then select a day without rain, finally book a flight from San Francisco.",21 "instruction": "Locate a travel guide for Bali Island using Tripadvisor, note down the resource website in Google Keep, check The Weather Channel for a rain-free day, and use Traveloka to book a flight departing from San Francisco."22 },23 "step_length": 34,24 "steps": [25 {26 "step": 0,27 "screenshot": "0055180990380237_0.png",28 "action": "CLICK",29 "info": [30 [31 683,32 21033 ],34 [35 683,36 21037 ]38 ],39 "ps": "[(683, 210)]"40 },41 {42 "step": 1,43 "screenshot": "0055180990380237_1.png",44 "action": "CLICK",45 "info": [46 [47 77,48 7449 ],50 [51 77,52 7453 ]54 ],55 "ps": "[(77, 74)]"56 },57 {58 "step": 2,59 "screenshot": "0055180990380237_2.png",60 "action": "CLICK",61 "info": [62 [63 863,64 11565 ],66 [67 863,68 11569 ]70 ],71 "ps": "[(863, 115)]"72 },73 {74 "step": 3,75 "screenshot": "0055180990380237_3.png",76 "action": "TEXT",77 "info": "Bali Island",78 "ps": ""79 },80 {81 "step": 4,82 "screenshot": "0055180990380237_4.png",83 "action": "CLICK",84 "info": [85 [86 513,87 39588 ],89 [90 513,91 39592 ]93 ],94 "ps": "[(513, 395)]"95 },96 {97 "step": 5,98 "screenshot": "0055180990380237_5.png",99 "action": "CLICK",100 "info": [101 [102 794,103 81104 ],105 [106 794,107 81108 ]109 ],110 "ps": "[(794, 81)]"111 },112 {113 "step": 6,114 "screenshot": "0055180990380237_6.png",115 "action": "CLICK",116 "info": [117 [118 880,119 501120 ],121 [122 880,123 501124 ]125 ],126 "ps": "[(880, 501)]"127 },128 {129 "step": 7,130 "screenshot": "0055180990380237_7.png",131 "action": "CLICK",132 "info": "KEY_HOME",133 "ps": ""134 },135 {136 "step": 8,137 "screenshot": "0055180990380237_8.png",138 "action": "SCROLL",139 "info": [140 [141 322,142 514143 ],144 [145 998,146 506147 ]148 ],149 "ps": "[(322, 514), (330, 514), (341, 514), (363, 514), (408, 514), (463, 514), (538, 514), (630, 514), (733, 514), (822, 514), (905, 510), (977, 507), (998, 506)]"150 },151 {152 "step": 9,153 "screenshot": "0055180990380237_9.png",154 "action": "CLICK",155 "info": [156 [157 141,158 335159 ],160 [161 141,162 335163 ]164 ],165 "ps": "[(141, 335)]"166 },167 {168 "step": 10,169 "screenshot": "0055180990380237_10.png",170 "action": "CLICK",171 "info": [172 [173 83,174 85175 ],176 [177 83,178 85179 ]180 ],181 "ps": "[(83, 85)]"182 },183 {184 "step": 11,185 "screenshot": "0055180990380237_11.png",186 "action": "CLICK",187 "info": [188 [189 858,190 862191 ],192 [193 858,194 862195 ]196 ],197 "ps": "[(858, 862)]"198 },199 {200 "step": 12,201 "screenshot": "0055180990380237_12.png",202 "action": "CLICK",203 "info": [204 [205 511,206 568207 ],208 [209 511,210 568211 ]212 ],213 "ps": "[(511, 568)]"214 },215 {216 "step": 13,217 "screenshot": "0055180990380237_13.png",218 "action": "CLICK",219 "info": [220 [221 205,222 187223 ],224 [225 205,226 187227 ]228 ],229 "ps": "[(205, 187)]"230 },231 {232 "step": 14,233 "screenshot": "0055180990380237_14.png",234 "action": "TEXT",235 "info": "a travel guide to Bali Island",236 "ps": ""237 },238 {239 "step": 15,240 "screenshot": "0055180990380237_15.png",241 "action": "CLICK",242 "info": "KEY_HOME",243 "ps": ""244 },245 {246 "step": 16,247 "screenshot": "0055180990380237_16.png",248 "action": "CLICK",249 "info": [250 [251 841,252 354253 ],254 [255 841,256 354257 ]258 ],259 "ps": "[(841, 354)]"260 },261 {262 "step": 17,263 "screenshot": "0055180990380237_17.png",264 "action": "CLICK",265 "info": [266 [267 274,268 82269 ],270 [271 274,272 82273 ]274 ],275 "ps": "[(274, 82)]"276 },277 {278 "step": 18,279 "screenshot": "0055180990380237_18.png",280 "action": "TEXT",281 "info": "Bali",282 "ps": ""283 },284 {285 "step": 19,286 "screenshot": "0055180990380237_19.png",287 "action": "TEXT",288 "info": " Island",289 "ps": ""290 },291 {292 "step": 20,293 "screenshot": "0055180990380237_20.png",294 "action": "CLICK",295 "info": [296 [297 336,298 190299 ],300 [301 336,302 190303 ]304 ],305 "ps": "[(336, 190)]"306 },307 {308 "step": 21,309 "screenshot": "0055180990380237_21.png",310 "action": "CLICK",311 "info": "KEY_HOME",312 "ps": ""313 },314 {315 "step": 22,316 "screenshot": "0055180990380237_22.png",317 "action": "SCROLL",318 "info": [319 [320 841,321 492322 ],323 [324 11,325 529326 ]327 ],328 "ps": "[(841, 492), (838, 492), (833, 492), (822, 492), (808, 492), (788, 492), (749, 492), (699, 492), (622, 495), (516, 498), (399, 506), (258, 514), (130, 521), (11, 529)]"329 },330 {331 "step": 23,332 "screenshot": "0055180990380237_23.png",333 "action": "CLICK",334 "info": [335 [336 341,337 209338 ],339 [340 341,341 209342 ]343 ],344 "ps": "[(341, 209)]"345 },346 {347 "step": 24,348 "screenshot": "0055180990380237_24.png",349 "action": "CLICK",350 "info": [351 [352 347,353 412354 ],355 [356 347,357 412358 ]359 ],360 "ps": "[(347, 412)]"361 },362 {363 "step": 25,364 "screenshot": "0055180990380237_25.png",365 "action": "CLICK",366 "info": [367 [368 316,369 93370 ],371 [372 316,373 93374 ]375 ],376 "ps": "[(316, 93)]"377 },378 {379 "step": 26,380 "screenshot": "0055180990380237_26.png",381 "action": "TEXT",382 "info": "Bali",383 "ps": ""384 },385 {386 "step": 27,387 "screenshot": "0055180990380237_27.png",388 "action": "CLICK",389 "info": [390 [391 277,392 231393 ],394 [395 277,396 231397 ]398 ],399 "ps": "[(277, 231)]"400 },401 {402 "step": 28,403 "screenshot": "0055180990380237_28.png",404 "action": "CLICK",405 "info": [406 [407 299,408 535409 ],410 [411 299,412 535413 ]414 ],415 "ps": "[(299, 535)]"416 },417 {418 "step": 29,419 "screenshot": "0055180990380237_29.png",420 "action": "CLICK",421 "info": [422 [423 927,424 617425 ],426 [427 927,428 617429 ]430 ],431 "ps": "[(927, 617)]"432 },433 {434 "step": 30,435 "screenshot": "0055180990380237_30.png",436 "action": "CLICK",437 "info": [438 [439 494,440 762441 ],442 [443 494,444 762445 ]446 ],447 "ps": "[(494, 762)]"448 },449 {450 "step": 31,451 "screenshot": "0055180990380237_31.png",452 "action": "CLICK",453 "info": [454 [455 505,456 432457 ],458 [459 505,460 432461 ]462 ],463 "ps": "[(505, 432)]"464 },465 {466 "step": 32,467 "screenshot": "0055180990380237_32.png",468 "action": "CLICK",469 "info": [470 [471 827,472 895473 ],474 [475 827,476 895477 ]478 ],479 "ps": "[(827, 895)]"480 },481 {482 "step": 33,483 "screenshot": "0055180990380237_33.png",484 "action": "COMPLETE",485 "info": "",486 "ps": "From San Francisco"487 }488 ]489}