OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0197987108916412",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1600,8 "w": 2560,9 "device_name": "Pixel Tablet"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Dropbox Paper",15 "Wikipedia",16 "Picturethis"17 ],18 "meta_task": "Find a photograph of a/an {}, identify it using a plant identifier app, and if the identification result is correct, record the image along with the plant name in a document.",19 "task": "Find a photograph of a/an Iris, identify it using a plant identifier app, and if the identification result is correct, record the image along with the plant name in a document.",20 "instruction": "Locate a photograph of an Iris on Wikipedia, utilize the PictureThis app to identify the plant, and upon confirming the identification, document the image and plant name in a Dropbox Paper file."21 },22 "step_length": 28,23 "steps": [24 {25 "step": 0,26 "screenshot": "0197987108916412_0.png",27 "action": "CLICK",28 "info": [29 [30 147,31 9332 ],33 [34 147,35 9336 ]37 ],38 "ps": "[(147, 93)]"39 },40 {41 "step": 1,42 "screenshot": "0197987108916412_1.png",43 "action": "CLICK",44 "info": [45 [46 344,47 13748 ],49 [50 344,51 13752 ]53 ],54 "ps": "[(344, 137)]"55 },56 {57 "step": 2,58 "screenshot": "0197987108916412_2.png",59 "action": "TEXT",60 "info": "Iris",61 "ps": ""62 },63 {64 "step": 3,65 "screenshot": "0197987108916412_3.png",66 "action": "TEXT",67 "info": "(plant)",68 "ps": ""69 },70 {71 "step": 4,72 "screenshot": "0197987108916412_4.png",73 "action": "CLICK",74 "info": [75 [76 104,77 16478 ],79 [80 104,81 16482 ]83 ],84 "ps": "[(104, 164)]"85 },86 {87 "step": 5,88 "screenshot": "0197987108916412_5.png",89 "action": "CLICK",90 "info": [91 [92 430,93 81294 ],95 [96 430,97 81298 ]99 ],100 "ps": "[(430, 812)]"101 },102 {103 "step": 6,104 "screenshot": "0197987108916412_6.png",105 "action": "SCROLL",106 "info": [107 [108 352,109 537110 ],111 [112 590,113 498114 ]115 ],116 "ps": "[(352, 537), (353, 537), (354, 537), (355, 537), (355, 537), (355, 536), (356, 536), (357, 536), (358, 536), (358, 536), (359, 536), (360, 536), (361, 536), (362, 534), (363, 534), (364, 534), (365, 534), (366, 534), (367, 534), (368, 534), (369, 534), (371, 534), (372, 534), (373, 533), (376, 533), (377, 533), (378, 533), (380, 533), (381, 532), (383, 532), (384, 532), (387, 532), (388, 532), (390, 531), (393, 531), (394, 531), (397, 528), (398, 528), (401, 528), (403, 528), (404, 528), (407, 528), (408, 527), (411, 527), (413, 527), (415, 527), (418, 527), (420, 527), (423, 526), (424, 526), (428, 526), (430, 526), (432, 526), (433, 526), (436, 526), (438, 526), (440, 526), (443, 526), (446, 526), (448, 526), (450, 526), (452, 526), (455, 524), (459, 524), (463, 524), (465, 524), (468, 524), (470, 524), (473, 524), (476, 523), (477, 523), (480, 523), (482, 523), (483, 523), (485, 523), (488, 522), (490, 522), (491, 522), (494, 522), (496, 521), (498, 521), (501, 521), (502, 521), (505, 519), (508, 519), (510, 519), (512, 518), (514, 518), (516, 518), (518, 517), (520, 517), (522, 517), (524, 516), (527, 516), (529, 516), (531, 513), (534, 513), (536, 512), (537, 512), (541, 511), (542, 511), (544, 511), (545, 509), (546, 509), (548, 508), (551, 508), (552, 508), (554, 507), (555, 507), (558, 507), (561, 506), (562, 506), (565, 504), (566, 504), (569, 503), (571, 503), (572, 503), (573, 503), (576, 502), (578, 502), (580, 501), (582, 501), (583, 501), (586, 501), (588, 499), (590, 498)]"117 },118 {119 "step": 7,120 "screenshot": "0197987108916412_7.png",121 "action": "CLICK",122 "info": [123 [124 980,125 52126 ],127 [128 980,129 52130 ]131 ],132 "ps": "[(980, 52)]"133 },134 {135 "step": 8,136 "screenshot": "0197987108916412_8.png",137 "action": "CLICK",138 "info": [139 [140 905,141 52142 ],143 [144 905,145 52146 ]147 ],148 "ps": "[(905, 52)]"149 },150 {151 "step": 9,152 "screenshot": "0197987108916412_9.png",153 "action": "CLICK",154 "info": "KEY_HOME",155 "ps": ""156 },157 {158 "step": 10,159 "screenshot": "0197987108916412_10.png",160 "action": "CLICK",161 "info": [162 [163 281,164 256165 ],166 [167 281,168 256169 ]170 ],171 "ps": "[(281, 256)]"172 },173 {174 "step": 11,175 "screenshot": "0197987108916412_11.png",176 "action": "CLICK",177 "info": [178 [179 685,180 93181 ],182 [183 685,184 93185 ]186 ],187 "ps": "[(685, 93)]"188 },189 {190 "step": 12,191 "screenshot": "0197987108916412_12.png",192 "action": "CLICK",193 "info": [194 [195 505,196 852197 ],198 [199 505,200 852201 ]202 ],203 "ps": "[(505, 852)]"204 },205 {206 "step": 13,207 "screenshot": "0197987108916412_13.png",208 "action": "CLICK",209 "info": [210 [211 387,212 829213 ],214 [215 387,216 829217 ]218 ],219 "ps": "[(387, 829)]"220 },221 {222 "step": 14,223 "screenshot": "0197987108916412_14.png",224 "action": "CLICK",225 "info": [226 [227 590,228 674229 ],230 [231 590,232 674233 ]234 ],235 "ps": "[(590, 674)]"236 },237 {238 "step": 15,239 "screenshot": "0197987108916412_15.png",240 "action": "SCROLL",241 "info": [242 [243 573,244 833245 ],246 [247 555,248 463249 ]250 ],251 "ps": "[(573, 833), (572, 833), (572, 832), (572, 831), (572, 829), (571, 829), (571, 828), (571, 827), (571, 826), (571, 824), (571, 823), (571, 822), (571, 821), (571, 819), (571, 818), (571, 817), (571, 816), (571, 814), (570, 813), (570, 812), (570, 811), (570, 809), (570, 808), (570, 807), (570, 806), (570, 804), (569, 804), (569, 803), (569, 802), (569, 801), (569, 799), (569, 798), (569, 797), (569, 796), (569, 794), (569, 793), (569, 792), (569, 791), (569, 789), (569, 788), (569, 787), (569, 786), (569, 784), (569, 783), (569, 782), (569, 781), (569, 779), (569, 778), (569, 777), (569, 776), (569, 774), (569, 773), (569, 772), (568, 772), (568, 771), (568, 769), (568, 768), (568, 767), (568, 766), (568, 764), (568, 763), (567, 762), (567, 761), (567, 759), (567, 758), (567, 757), (567, 756), (567, 754), (567, 753), (567, 752), (567, 749), (567, 748), (567, 747), (567, 746), (567, 744), (567, 743), (567, 742), (567, 741), (566, 741), (566, 739), (566, 738), (566, 737), (566, 736), (566, 734), (566, 733), (566, 732), (566, 731), (566, 729), (566, 728), (566, 727), (566, 726), (565, 711), (565, 709), (565, 708), (565, 707), (565, 706), (565, 704), (565, 702), (565, 701), (565, 699), (565, 698), (565, 697), (565, 696), (565, 694), (565, 693), (565, 692), (565, 691), (565, 688), (565, 686), (565, 683), (564, 681), (564, 679), (564, 678), (564, 677), (564, 676), (564, 674), (564, 673), (564, 672), (564, 671), (564, 669), (563, 668), (563, 667), (563, 666), (563, 664), (563, 663), (563, 662), (563, 661), (563, 659), (563, 658), (563, 657), (563, 656), (563, 654), (563, 653), (563, 652), (562, 651), (562, 649), (562, 648), (562, 646), (562, 643), (562, 642), (562, 641), (562, 639), (562, 638), (562, 637), (562, 636), (562, 634), (562, 633), (562, 631), (562, 628), (562, 627), (562, 623), (562, 621), (562, 618), (561, 617), (561, 616), (561, 613), (561, 612), (561, 611), (561, 609), (561, 607), (561, 606), (561, 604), (561, 603), (561, 602), (561, 599), (561, 598), (561, 597), (561, 596), (561, 594), (560, 593), (560, 592), (560, 589), (560, 588), (560, 587), (560, 586), (560, 583), (560, 582), (560, 581), (560, 579), (560, 578), (560, 577), (560, 576), (560, 574), (559, 573), (559, 572), (559, 571), (559, 569), (559, 568), (559, 567), (559, 566), (559, 564), (559, 563), (559, 562), (559, 559), (559, 558), (559, 557), (559, 556), (558, 553), (558, 552), (558, 551), (558, 549), (558, 548), (558, 547), (558, 546), (558, 544), (558, 543), (558, 541), (558, 539), (558, 538), (558, 537), (558, 536), (558, 534), (558, 533), (558, 532), (558, 531), (558, 529), (558, 528), (558, 527), (558, 526), (558, 524), (558, 523), (558, 522), (558, 519), (557, 518), (557, 517), (557, 516), (557, 514), (557, 513), (557, 512), (557, 509), (557, 508), (557, 507), (557, 506), (557, 504), (557, 503), (557, 502), (557, 501), (557, 499), (556, 498), (556, 496), (556, 494), (556, 493), (556, 492), (556, 491), (556, 489), (556, 488), (556, 487), (556, 486), (556, 484), (556, 483), (556, 482), (556, 481), (556, 479), (556, 478), (556, 477), (556, 476), (556, 474), (556, 473), (556, 472), (556, 471), (555, 471), (555, 469), (555, 468), (555, 467), (555, 466), (555, 464), (555, 463)]"252 },253 {254 "step": 16,255 "screenshot": "0197987108916412_16.png",256 "action": "CLICK",257 "info": [258 [259 600,260 332261 ],262 [263 600,264 332265 ]266 ],267 "ps": "[(600, 332)]"268 },269 {270 "step": 17,271 "screenshot": "0197987108916412_17.png",272 "action": "CLICK",273 "info": [274 [275 500,276 828277 ],278 [279 500,280 828281 ]282 ],283 "ps": "[(500, 828)]"284 },285 {286 "step": 18,287 "screenshot": "0197987108916412_18.png",288 "action": "CLICK",289 "info": "KEY_HOME",290 "ps": ""291 },292 {293 "step": 19,294 "screenshot": "0197987108916412_19.png",295 "action": "CLICK",296 "info": [297 [298 130,299 408300 ],301 [302 130,303 408304 ]305 ],306 "ps": "[(130, 408)]"307 },308 {309 "step": 20,310 "screenshot": "0197987108916412_20.png",311 "action": "CLICK",312 "info": [313 [314 273,315 884316 ],317 [318 273,319 884320 ]321 ],322 "ps": "[(273, 884)]"323 },324 {325 "step": 21,326 "screenshot": "0197987108916412_21.png",327 "action": "CLICK",328 "info": [329 [330 494,331 237332 ],333 [334 494,335 237336 ]337 ],338 "ps": "[(494, 237)]"339 },340 {341 "step": 22,342 "screenshot": "0197987108916412_22.png",343 "action": "TEXT",344 "info": "Iris",345 "ps": ""346 },347 {348 "step": 23,349 "screenshot": "0197987108916412_23.png",350 "action": "CLICK",351 "info": [352 [353 408,354 429355 ],356 [357 408,358 429359 ]360 ],361 "ps": "[(408, 429)]"362 },363 {364 "step": 24,365 "screenshot": "0197987108916412_24.png",366 "action": "CLICK",367 "info": [368 [369 453,370 548371 ],372 [373 453,374 548375 ]376 ],377 "ps": "[(453, 548)]"378 },379 {380 "step": 25,381 "screenshot": "0197987108916412_25.png",382 "action": "CLICK",383 "info": [384 [385 229,386 459387 ],388 [389 229,390 459391 ]392 ],393 "ps": "[(229, 459)]"394 },395 {396 "step": 26,397 "screenshot": "0197987108916412_26.png",398 "action": "CLICK",399 "info": [400 [401 336,402 82403 ],404 [405 336,406 82407 ]408 ],409 "ps": "[(336, 82)]"410 },411 {412 "step": 27,413 "screenshot": "0197987108916412_27.png",414 "action": "COMPLETE",415 "info": "",416 "ps": ""417 }418 ]419}