OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0541084014583131",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 1280,8 "w": 720,9 "device_name": "Small Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Things",15 "Ebay",16 "Firefox Browser"17 ],18 "meta_task": "Find a recipe for {}, create a shopping list of main ingredients, and add to the cart.",19 "task": "Find a recipe for Chocolate chip cookies, create a shopping list of main ingredients, and add to the cart.",20 "instruction": "Using the Firefox Browser, search for a recipe for Chocolate Chip Cookies. Then, compile a shopping list of the main ingredients in the Things app. Finally, add these items to your cart on eBay."21 },22 "step_length": 24,23 "steps": [24 {25 "step": 0,26 "screenshot": "0541084014583131_0.png",27 "action": "CLICK",28 "info": [29 [30 611,31 73232 ],33 [34 611,35 73236 ]37 ],38 "ps": "[(611, 732)]"39 },40 {41 "step": 1,42 "screenshot": "0541084014583131_1.png",43 "action": "CLICK",44 "info": "KEY_HOME",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "0541084014583131_2.png",50 "action": "CLICK",51 "info": [52 [53 174,54 32055 ],56 [57 174,58 32059 ]60 ],61 "ps": "[(174, 320)]"62 },63 {64 "step": 3,65 "screenshot": "0541084014583131_3.png",66 "action": "CLICK",67 "info": [68 [69 880,70 89071 ],72 [73 880,74 89075 ]76 ],77 "ps": "[(880, 890)]"78 },79 {80 "step": 4,81 "screenshot": "0541084014583131_4.png",82 "action": "CLICK",83 "info": [84 [85 422,86 44587 ],88 [89 422,90 44591 ]92 ],93 "ps": "[(422, 445)]"94 },95 {96 "step": 5,97 "screenshot": "0541084014583131_5.png",98 "action": "TEXT",99 "info": "Chocolate chip cookies: 150g salted butter, softened 80g light brown muscovado sugar, 80g granulated sugar, 2 tsp vanilla extract, 1 large egg, 225g plain flour, 1/2 tsp bicarbonate of soda, 1/4 tsp salt, 200g plain chocolate chips or chunks",100 "ps": ""101 },102 {103 "step": 6,104 "screenshot": "0541084014583131_6.png",105 "action": "CLICK",106 "info": [107 [108 502,109 478110 ],111 [112 502,113 478114 ]115 ],116 "ps": "[(502, 478)]"117 },118 {119 "step": 7,120 "screenshot": "0541084014583131_7.png",121 "action": "CLICK",122 "info": "KEY_HOME",123 "ps": ""124 },125 {126 "step": 8,127 "screenshot": "0541084014583131_8.png",128 "action": "CLICK",129 "info": [130 [131 363,132 451133 ],134 [135 363,136 451137 ]138 ],139 "ps": "[(363, 451)]"140 },141 {142 "step": 9,143 "screenshot": "0541084014583131_9.png",144 "action": "CLICK",145 "info": [146 [147 52,148 64149 ],150 [151 52,152 64153 ]154 ],155 "ps": "[(52, 64)]"156 },157 {158 "step": 10,159 "screenshot": "0541084014583131_10.png",160 "action": "CLICK",161 "info": [162 [163 502,164 90165 ],166 [167 502,168 90169 ]170 ],171 "ps": "[(502, 90)]"172 },173 {174 "step": 11,175 "screenshot": "0541084014583131_11.png",176 "action": "CLICK",177 "info": [178 [179 919,180 903181 ],182 [183 919,184 903185 ]186 ],187 "ps": "[(919, 903)]"188 },189 {190 "step": 12,191 "screenshot": "0541084014583131_12.png",192 "action": "CLICK",193 "info": [194 [195 649,196 351197 ],198 [199 649,200 351201 ]202 ],203 "ps": "[(649, 351)]"204 },205 {206 "step": 13,207 "screenshot": "0541084014583131_13.png",208 "action": "SCROLL",209 "info": [210 [211 616,212 884213 ],214 [215 738,216 174217 ]218 ],219 "ps": "[(616, 884), (616, 882), (616, 879), (616, 878), (622, 874), (622, 871), (627, 871), (627, 870), (627, 867), (630, 867), (630, 865), (636, 862), (636, 859), (636, 857), (641, 857), (641, 854), (641, 851), (644, 851), (644, 849), (644, 846), (649, 845), (649, 842), (652, 839), (652, 837), (652, 834), (652, 832), (658, 832), (658, 829), (658, 826), (658, 824), (658, 821), (663, 821), (663, 818), (663, 817), (663, 814), (666, 814), (666, 812), (666, 809), (666, 806), (666, 804), (666, 801), (666, 798), (672, 796), (672, 793), (672, 792), (672, 789), (672, 785), (672, 784), (672, 781), (672, 778), (672, 776), (674, 773), (674, 771), (674, 768), (674, 765), (674, 764), (674, 760), (674, 757), (674, 756), (674, 753), (674, 751), (674, 748), (674, 745), (674, 743), (674, 740), (674, 739), (674, 735), (674, 732), (674, 731), (674, 728), (674, 724), (674, 723), (674, 720), (674, 718), (674, 715), (674, 712), (674, 710), (674, 707), (674, 704), (674, 703), (674, 699), (674, 698), (674, 695), (674, 692), (674, 690), (674, 687), (674, 684), (674, 679), (674, 676), (674, 674), (674, 671), (674, 670), (674, 667), (674, 664), (674, 662), (674, 659), (674, 656), (674, 654), (674, 651), (674, 649), (674, 646), (674, 643), (674, 642), (674, 639), (674, 635), (674, 634), (674, 631), (674, 629), (674, 626), (674, 623), (674, 621), (674, 618), (674, 615), (674, 614), (674, 610), (674, 609), (674, 606), (674, 603), (674, 601), (674, 598), (674, 596), (674, 593), (674, 590), (674, 589), (674, 585), (674, 582), (674, 581), (674, 578), (674, 576), (674, 573), (674, 570), (674, 568), (674, 565), (674, 562), (669, 560), (669, 557), (669, 556), (669, 553), (669, 549), (669, 548), (669, 545), (669, 542), (669, 540), (669, 537), (669, 535), (669, 532), (669, 529), (669, 528), (669, 524), (669, 523), (669, 520), (669, 517), (669, 515), (669, 512), (669, 509), (669, 507), (669, 504), (669, 503), (669, 499), (669, 496), (669, 495), (669, 492), (669, 489), (669, 487), (669, 484), (669, 482), (669, 479), (669, 476), (669, 474), (669, 471), (669, 468), (669, 467), (669, 464), (669, 462), (669, 459), (669, 456), (669, 454), (669, 451), (669, 448), (669, 446), (669, 443), (669, 442), (669, 439), (669, 435), (669, 434), (669, 431), (669, 429), (669, 426), (669, 423), (669, 421), (669, 418), (669, 415), (669, 414), (669, 410), (669, 409), (669, 406), (669, 403), (669, 401), (669, 398), (669, 395), (669, 393), (669, 390), (669, 389), (669, 385), (669, 382), (669, 381), (669, 378), (669, 374), (669, 373), (669, 370), (669, 368), (672, 368), (672, 365), (672, 362), (672, 360), (672, 357), (672, 354), (672, 353), (674, 349), (674, 348), (674, 345), (674, 342), (674, 340), (674, 337), (674, 335), (674, 332), (680, 329), (680, 328), (680, 324), (680, 321), (680, 320), (686, 317), (686, 315), (686, 312), (686, 309), (686, 307), (686, 304), (686, 301), (688, 301), (688, 299), (688, 296), (688, 295), (688, 292), (688, 289), (688, 287), (694, 287), (694, 284), (694, 281), (694, 279), (694, 276), (694, 274), (699, 274), (699, 271), (699, 268), (699, 267), (699, 264), (699, 262), (702, 259), (702, 256), (702, 254), (702, 251), (702, 248), (702, 246), (708, 246), (708, 243), (708, 242), (708, 239), (711, 239), (711, 235), (711, 234), (711, 231), (711, 228), (716, 226), (716, 223), (716, 221), (722, 221), (722, 218), (722, 215), (722, 214), (722, 210), (724, 210), (724, 207), (724, 206), (724, 203), (724, 201), (730, 201), (730, 198), (730, 195), (730, 193), (730, 190), (736, 190), (736, 187), (736, 185), (738, 185), (738, 182), (738, 181), (738, 178), (738, 174)]"220 },221 {222 "step": 14,223 "screenshot": "0541084014583131_14.png",224 "action": "CLICK",225 "info": [226 [227 172,228 634229 ],230 [231 172,232 634233 ]234 ],235 "ps": "[(172, 634)]"236 },237 {238 "step": 15,239 "screenshot": "0541084014583131_15.png",240 "action": "CLICK",241 "info": [242 [243 611,244 121245 ],246 [247 611,248 121249 ]250 ],251 "ps": "[(611, 121)]"252 },253 {254 "step": 16,255 "screenshot": "0541084014583131_16.png",256 "action": "CLICK",257 "info": [258 [259 494,260 89261 ],262 [263 494,264 89265 ]266 ],267 "ps": "[(494, 89)]"268 },269 {270 "step": 17,271 "screenshot": "0541084014583131_17.png",272 "action": "CLICK",273 "info": [274 [275 738,276 92277 ],278 [279 738,280 92281 ]282 ],283 "ps": "[(738, 92)]"284 },285 {286 "step": 18,287 "screenshot": "0541084014583131_18.png",288 "action": "TEXT",289 "info": "vanilla extract",290 "ps": ""291 },292 {293 "step": 19,294 "screenshot": "0541084014583131_19.png",295 "action": "CLICK",296 "info": [297 [298 919,299 901300 ],301 [302 919,303 901304 ]305 ],306 "ps": "[(919, 901)]"307 },308 {309 "step": 20,310 "screenshot": "0541084014583131_20.png",311 "action": "CLICK",312 "info": [313 [314 711,315 357316 ],317 [318 711,319 357320 ]321 ],322 "ps": "[(711, 357)]"323 },324 {325 "step": 21,326 "screenshot": "0541084014583131_21.png",327 "action": "SCROLL",328 "info": [329 [330 477,331 842332 ],333 [334 616,335 45336 ]337 ],338 "ps": "[(477, 842), (477, 840), (477, 839), (477, 835), (483, 835), (483, 834), (488, 834), (488, 831), (491, 828), (491, 826), (497, 823), (497, 820), (502, 818), (505, 815), (505, 814), (511, 810), (511, 807), (513, 806), (513, 803), (519, 799), (519, 798), (524, 795), (524, 793), (527, 790), (527, 787), (533, 785), (533, 779), (538, 779), (538, 774), (544, 768), (544, 762), (547, 760), (547, 757), (547, 754), (552, 753), (552, 749), (555, 748), (555, 745), (555, 742), (561, 740), (561, 734), (561, 732), (566, 729), (566, 726), (566, 724), (566, 721), (569, 720), (569, 717), (569, 714), (574, 712), (574, 706), (574, 704), (574, 701), (577, 698), (577, 696), (577, 693), (577, 690), (583, 689), (583, 685), (583, 681), (583, 678), (588, 676), (588, 673), (588, 670), (588, 668), (588, 665), (591, 662), (591, 660), (591, 657), (591, 656), (597, 653), (597, 649), (597, 648), (597, 642), (597, 637), (602, 631), (602, 628), (602, 624), (602, 623), (602, 620), (602, 618), (602, 615), (602, 612), (602, 610), (602, 607), (602, 606), (608, 603), (608, 599), (608, 598), (608, 595), (608, 592), (608, 590), (608, 587), (608, 585), (608, 582), (608, 579), (608, 578), (608, 574), (608, 571), (608, 570), (608, 567), (608, 565), (611, 562), (611, 559), (611, 557), (611, 554), (611, 551), (611, 549), (611, 546), (611, 545), (611, 542), (611, 539), (611, 537), (611, 534), (611, 532), (611, 529), (611, 526), (611, 524), (611, 521), (611, 518), (611, 517), (611, 514), (611, 512), (611, 509), (611, 506), (611, 504), (611, 501), (611, 498), (611, 496), (611, 493), (611, 492), (611, 489), (611, 485), (611, 484), (611, 481), (611, 478), (611, 476), (611, 473), (611, 471), (611, 468), (611, 465), (611, 464), (611, 460), (611, 457), (611, 456), (611, 453), (611, 451), (611, 448), (611, 445), (611, 443), (611, 440), (611, 439), (611, 435), (611, 432), (611, 431), (611, 428), (611, 424), (611, 423), (611, 420), (611, 418), (611, 415), (611, 412), (611, 410), (611, 407), (611, 404), (611, 403), (611, 399), (611, 398), (611, 395), (611, 392), (611, 390), (611, 387), (611, 384), (611, 382), (611, 379), (611, 378), (611, 374), (611, 371), (611, 370), (611, 367), (611, 364), (611, 362), (611, 359), (611, 357), (611, 354), (611, 351), (611, 349), (611, 346), (611, 345), (611, 342), (611, 339), (611, 337), (611, 334), (611, 331), (611, 329), (611, 326), (611, 324), (611, 321), (611, 318), (611, 317), (611, 314), (611, 310), (611, 309), (611, 306), (611, 304), (611, 301), (611, 298), (611, 296), (611, 293), (611, 290), (611, 289), (611, 285), (611, 284), (611, 281), (611, 278), (611, 276), (611, 273), (611, 271), (611, 268), (611, 265), (611, 264), (611, 260), (611, 257), (611, 256), (611, 253), (611, 251), (611, 248), (605, 248), (605, 245), (605, 243), (605, 240), (605, 237), (605, 235), (605, 232), (605, 231), (605, 228), (605, 224), (605, 223), (605, 220), (605, 217), (605, 215), (605, 212), (605, 210), (605, 207), (605, 204), (605, 203), (605, 199), (605, 196), (605, 195), (605, 192), (605, 190), (605, 187), (605, 184), (605, 182), (605, 179), (605, 178), (605, 174), (605, 171), (605, 170), (605, 167), (605, 164), (605, 162), (605, 159), (605, 157), (605, 154), (605, 151), (605, 149), (605, 146), (605, 143), (605, 142), (605, 139), (605, 137), (605, 134), (605, 131), (605, 129), (605, 126), (605, 123), (605, 121), (605, 118), (605, 117), (605, 114), (605, 110), (605, 109), (605, 106), (605, 103), (608, 103), (608, 101), (608, 98), (608, 96), (608, 93), (608, 90), (608, 89), (611, 89), (611, 85), (611, 84), (611, 81), (611, 78), (611, 76), (611, 73), (611, 70), (611, 68), (611, 65), (611, 64), (611, 60), (611, 57), (611, 56), (616, 56), (616, 53), (616, 49), (616, 48), (616, 45)]"339 },340 {341 "step": 22,342 "screenshot": "0541084014583131_22.png",343 "action": "CLICK",344 "info": [345 [346 486,347 898348 ],349 [350 486,351 898352 ]353 ],354 "ps": "[(486, 898)]"355 },356 {357 "step": 23,358 "screenshot": "0541084014583131_23.png",359 "action": "COMPLETE",360 "info": "",361 "ps": ""362 }363 ]364}