OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "1177930630437411",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1840,8 "w": 2208,9 "device_name": "Pixel Fold"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "DuckDuckGo",15 "Ebay",16 "Facebook"17 ],18 "meta_task": "Search for a famous {} book, read reviews, and purchase the book.",19 "task": "Search for a famous cookbook book, read reviews, and purchase the book.",20 "instruction": "Utilize DuckDuckGo to search for a renowned cookbook, check out reviews on Facebook, and finally, make the purchase on Ebay."21 },22 "step_length": 24,23 "steps": [24 {25 "step": 0,26 "screenshot": "1177930630437411_0.png",27 "action": "CLICK",28 "info": [29 [30 794,31 32632 ],33 [34 794,35 32636 ]37 ],38 "ps": "[(794, 326)]"39 },40 {41 "step": 1,42 "screenshot": "1177930630437411_1.png",43 "action": "TEXT",44 "info": "book about cookbook",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "1177930630437411_2.png",50 "action": "CLICK",51 "info": [52 [53 863,54 87855 ],56 [57 863,58 87859 ]60 ],61 "ps": "[(863, 878)]"62 },63 {64 "step": 3,65 "screenshot": "1177930630437411_3.png",66 "action": "SCROLL",67 "info": [68 [69 441,70 81271 ],72 [73 464,74 39775 ]76 ],77 "ps": "[(441, 812), (441, 805), (441, 800), (441, 792), (441, 784), (441, 769), (441, 753), (441, 740), (441, 729), (441, 713), (441, 695), (441, 675), (444, 660), (445, 639), (445, 623), (445, 610), (447, 596), (447, 580), (447, 572), (448, 554), (448, 546), (451, 535), (451, 526), (451, 515), (452, 507), (452, 499), (452, 494), (452, 487), (452, 482), (452, 470), (454, 463), (454, 459), (454, 451), (455, 447), (457, 441), (457, 438), (458, 433), (458, 429), (458, 423), (458, 419), (461, 415), (461, 410), (462, 409), (462, 405), (462, 403), (462, 401), (462, 398), (462, 397), (464, 397)]"78 },79 {80 "step": 4,81 "screenshot": "1177930630437411_4.png",82 "action": "CLICK",83 "info": [84 [85 287,86 49487 ],88 [89 287,90 49491 ]92 ],93 "ps": "[(287, 494)]"94 },95 {96 "step": 5,97 "screenshot": "1177930630437411_5.png",98 "action": "SCROLL",99 "info": [100 [101 492,102 824103 ],104 [105 489,106 280107 ]108 ],109 "ps": "[(492, 824), (491, 824), (491, 821), (491, 817), (489, 812), (489, 809), (489, 804), (489, 796), (488, 784), (485, 776), (485, 767), (485, 753), (485, 744), (485, 729), (485, 715), (485, 695), (485, 680), (484, 663), (484, 640), (484, 627), (484, 616), (482, 590), (482, 576), (482, 563), (481, 547), (481, 538), (481, 518), (481, 506), (481, 490), (481, 475), (481, 466), (481, 451), (481, 441), (481, 426), (481, 421), (481, 417), (481, 405), (481, 401), (481, 393), (481, 386), (481, 378), (481, 370), (481, 365), (482, 353), (482, 346), (482, 342), (482, 337), (484, 333), (484, 329), (484, 322), (485, 318), (485, 310), (485, 306), (485, 304), (485, 301), (488, 298), (488, 297), (488, 294), (488, 292), (488, 290), (488, 289), (489, 286), (489, 284), (489, 282), (489, 280)]"110 },111 {112 "step": 6,113 "screenshot": "1177930630437411_6.png",114 "action": "SCROLL",115 "info": [116 [117 441,118 846119 ],120 [121 405,122 226123 ]124 ],125 "ps": "[(441, 846), (441, 844), (441, 840), (441, 836), (441, 832), (441, 826), (441, 820), (438, 813), (438, 804), (438, 792), (435, 780), (435, 767), (432, 753), (432, 732), (428, 715), (428, 699), (425, 679), (422, 660), (422, 643), (418, 627), (417, 608), (415, 590), (414, 570), (412, 554), (412, 546), (410, 518), (408, 502), (408, 494), (407, 478), (405, 463), (405, 455), (405, 441), (403, 429), (403, 415), (401, 405), (401, 395), (401, 381), (401, 374), (401, 366), (401, 358), (400, 350), (400, 345), (400, 341), (400, 333), (400, 329), (400, 321), (400, 317), (400, 314), (400, 313), (400, 309), (400, 306), (400, 304), (400, 302), (400, 301), (400, 298), (400, 297), (400, 292), (400, 290), (400, 289), (400, 286), (400, 284), (400, 280), (400, 278), (400, 277), (400, 272), (400, 270), (401, 269), (401, 266), (401, 264), (401, 261), (401, 260), (401, 258), (401, 256), (401, 254), (401, 252), (401, 249), (401, 248), (401, 246), (401, 244), (401, 241), (401, 240), (401, 238), (401, 236), (403, 236), (403, 233), (403, 232), (403, 229), (403, 228), (405, 228), (405, 226)]"126 },127 {128 "step": 7,129 "screenshot": "1177930630437411_7.png",130 "action": "SCROLL",131 "info": [132 [133 384,134 856135 ],136 [137 314,138 260139 ]140 ],141 "ps": "[(384, 856), (384, 852), (384, 848), (381, 840), (380, 832), (378, 820), (374, 801), (374, 785), (370, 764), (368, 744), (365, 727), (361, 692), (355, 671), (353, 652), (350, 635), (348, 627), (344, 608), (343, 596), (341, 583), (340, 570), (338, 558), (336, 543), (334, 530), (334, 518), (331, 506), (330, 487), (327, 478), (327, 466), (324, 454), (324, 443), (324, 429), (321, 415), (321, 405), (321, 393), (321, 385), (317, 370), (317, 354), (317, 341), (317, 324), (317, 314), (317, 302), (317, 294), (317, 286), (317, 280), (317, 274), (317, 270), (317, 269), (317, 264), (317, 261), (317, 260), (314, 260)]"142 },143 {144 "step": 8,145 "screenshot": "1177930630437411_8.png",146 "action": "CLICK",147 "info": "KEY_HOME",148 "ps": ""149 },150 {151 "step": 9,152 "screenshot": "1177930630437411_9.png",153 "action": "CLICK",154 "info": [155 [156 907,157 306158 ],159 [160 907,161 306162 ]163 ],164 "ps": "[(907, 306)]"165 },166 {167 "step": 10,168 "screenshot": "1177930630437411_10.png",169 "action": "CLICK",170 "info": [171 [172 712,173 71174 ],175 [176 712,177 71178 ]179 ],180 "ps": "[(712, 71)]"181 },182 {183 "step": 11,184 "screenshot": "1177930630437411_11.png",185 "action": "TEXT",186 "info": "the french laundry cookbook",187 "ps": ""188 },189 {190 "step": 12,191 "screenshot": "1177930630437411_12.png",192 "action": "CLICK",193 "info": [194 [195 860,196 897197 ],198 [199 860,200 897201 ]202 ],203 "ps": "[(860, 897)]"204 },205 {206 "step": 13,207 "screenshot": "1177930630437411_13.png",208 "action": "SCROLL",209 "info": [210 [211 420,212 711213 ],214 [215 424,216 309217 ]218 ],219 "ps": "[(420, 711), (420, 707), (420, 703), (420, 701), (420, 695), (420, 689), (420, 683), (420, 675), (420, 668), (420, 660), (420, 644), (420, 635), (420, 624), (420, 616), (420, 607), (420, 598), (420, 583), (420, 574), (420, 563), (420, 546), (420, 534), (420, 526), (420, 507), (420, 498), (420, 486), (420, 478), (420, 461), (420, 449), (420, 441), (420, 433), (420, 429), (420, 421), (420, 413), (420, 401), (420, 393), (420, 385), (420, 378), (420, 373), (420, 366), (420, 361), (420, 357), (420, 349), (420, 345), (422, 341), (422, 337), (422, 334), (422, 333), (422, 330), (422, 329), (422, 326), (422, 324), (422, 322), (422, 321), (422, 318), (422, 317), (422, 314), (424, 313), (424, 310), (424, 309)]"220 },221 {222 "step": 14,223 "screenshot": "1177930630437411_14.png",224 "action": "CLICK",225 "info": [226 [227 418,228 840229 ],230 [231 418,232 840233 ]234 ],235 "ps": "[(418, 840)]"236 },237 {238 "step": 15,239 "screenshot": "1177930630437411_15.png",240 "action": "CLICK",241 "info": "KEY_HOME",242 "ps": ""243 },244 {245 "step": 16,246 "screenshot": "1177930630437411_16.png",247 "action": "CLICK",248 "info": [249 [250 797,251 129252 ],253 [254 797,255 129256 ]257 ],258 "ps": "[(797, 129)]"259 },260 {261 "step": 17,262 "screenshot": "1177930630437411_17.png",263 "action": "CLICK",264 "info": [265 [266 405,267 172268 ],269 [270 405,271 172272 ]273 ],274 "ps": "[(405, 172)]"275 },276 {277 "step": 18,278 "screenshot": "1177930630437411_18.png",279 "action": "TEXT",280 "info": "the french laundry cookbook",281 "ps": ""282 },283 {284 "step": 19,285 "screenshot": "1177930630437411_19.png",286 "action": "CLICK",287 "info": [288 [289 863,290 884291 ],292 [293 861,294 884295 ]296 ],297 "ps": "[(863, 884), (861, 884)]"298 },299 {300 "step": 20,301 "screenshot": "1177930630437411_20.png",302 "action": "CLICK",303 "info": [304 [305 479,306 393307 ],308 [309 479,310 393311 ]312 ],313 "ps": "[(479, 393)]"314 },315 {316 "step": 21,317 "screenshot": "1177930630437411_21.png",318 "action": "SCROLL",319 "info": [320 [321 532,322 877323 ],324 [325 485,326 213327 ]328 ],329 "ps": "[(532, 877), (532, 872), (532, 869), (532, 864), (532, 858), (532, 852), (532, 846), (532, 840), (532, 832), (532, 826), (532, 820), (531, 812), (531, 801), (529, 788), (529, 773), (529, 764), (529, 755), (528, 741), (528, 729), (526, 713), (526, 695), (522, 671), (522, 656), (522, 632), (519, 616), (519, 600), (515, 580), (512, 564), (512, 550), (512, 538), (509, 526), (505, 502), (505, 490), (504, 475), (502, 461), (502, 454), (501, 441), (499, 423), (499, 415), (498, 401), (498, 390), (495, 385), (495, 373), (495, 363), (492, 349), (492, 337), (492, 329), (492, 324), (491, 318), (491, 314), (491, 309), (491, 304), (489, 301), (489, 297), (489, 292), (489, 289), (489, 284), (488, 277), (488, 274), (488, 272), (485, 269), (485, 264), (485, 261), (485, 260), (485, 258), (485, 252), (485, 248), (485, 244), (485, 240), (485, 236), (485, 232), (485, 226), (485, 224), (485, 220), (485, 217), (485, 216), (485, 213)]"330 },331 {332 "step": 22,333 "screenshot": "1177930630437411_22.png",334 "action": "CLICK",335 "info": [336 [337 485,338 659339 ],340 [341 485,342 659343 ]344 ],345 "ps": "[(485, 659)]"346 },347 {348 "step": 23,349 "screenshot": "1177930630437411_23.png",350 "action": "COMPLETE",351 "info": "",352 "ps": ""353 }354 ]355}