OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "1181506905418460",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Opera",15 "To-Do List",16 "Ebay"17 ],18 "meta_task": "Find a recipe for {}, create a shopping list of main ingredients, and add to the cart.",19 "task": "Find a recipe for Chocolate chip cookies, create a shopping list of main ingredients, and add to the cart.",20 "instruction": "Utilize Opera to locate a recipe for Chocolate Chip Cookies. Then, compile a shopping list of the essential ingredients using To-Do List. Finally, add these ingredients to your cart on eBay."21 },22 "step_length": 22,23 "steps": [24 {25 "step": 0,26 "screenshot": "1181506905418460_0.png",27 "action": "CLICK",28 "info": [29 [30 605,31 80732 ],33 [34 605,35 80736 ]37 ],38 "ps": "[(605, 807)]"39 },40 {41 "step": 1,42 "screenshot": "1181506905418460_1.png",43 "action": "CLICK",44 "info": "KEY_HOME",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "1181506905418460_2.png",50 "action": "CLICK",51 "info": [52 [53 885,54 23455 ],56 [57 885,58 23459 ]60 ],61 "ps": "[(885, 234)]"62 },63 {64 "step": 3,65 "screenshot": "1181506905418460_3.png",66 "action": "TEXT",67 "info": "Chocolate chip cookies: 150g salted butter, softened 80g light brown muscovado sugar, 80g granulated sugar, 2 tsp vanilla extract, 1 large egg, 225g plain flour, 1/2 tsp bicarbonate of soda, 1/4 tsp salt, 200g plain chocolate chips or chunks",68 "ps": ""69 },70 {71 "step": 4,72 "screenshot": "1181506905418460_4.png",73 "action": "CLICK",74 "info": [75 [76 928,77 60378 ],79 [80 928,81 60382 ]83 ],84 "ps": "[(928, 603)]"85 },86 {87 "step": 5,88 "screenshot": "1181506905418460_5.png",89 "action": "CLICK",90 "info": "KEY_HOME",91 "ps": ""92 },93 {94 "step": 6,95 "screenshot": "1181506905418460_6.png",96 "action": "CLICK",97 "info": [98 [99 349,100 378101 ],102 [103 349,104 378105 ]106 ],107 "ps": "[(349, 378)]"108 },109 {110 "step": 7,111 "screenshot": "1181506905418460_7.png",112 "action": "CLICK",113 "info": [114 [115 77,116 55117 ],118 [119 77,120 55121 ]122 ],123 "ps": "[(77, 55)]"124 },125 {126 "step": 8,127 "screenshot": "1181506905418460_8.png",128 "action": "CLICK",129 "info": [130 [131 556,132 51133 ],134 [135 556,136 51137 ]138 ],139 "ps": "[(556, 51)]"140 },141 {142 "step": 9,143 "screenshot": "1181506905418460_9.png",144 "action": "CLICK",145 "info": [146 [147 926,148 915149 ],150 [151 926,152 915153 ]154 ],155 "ps": "[(926, 915)]"156 },157 {158 "step": 10,159 "screenshot": "1181506905418460_10.png",160 "action": "CLICK",161 "info": [162 [163 671,164 632165 ],166 [167 671,168 632169 ]170 ],171 "ps": "[(671, 632)]"172 },173 {174 "step": 11,175 "screenshot": "1181506905418460_11.png",176 "action": "SCROLL",177 "info": [178 [179 462,180 743181 ],182 [183 498,184 283185 ]186 ],187 "ps": "[(462, 743), (462, 672), (462, 668), (462, 664), (462, 662), (462, 659), (462, 654), (462, 652), (462, 649), (462, 645), (462, 642), (462, 637), (462, 633), (462, 632), (462, 627), (462, 624), (462, 619), (462, 615), (462, 612), (462, 607), (462, 603), (462, 600), (462, 596), (462, 591), (462, 588), (462, 584), (462, 580), (462, 576), (462, 572), (462, 570), (462, 566), (462, 563), (462, 558), (462, 554), (463, 549), (463, 546), (463, 542), (463, 537), (463, 534), (463, 530), (463, 526), (463, 522), (463, 518), (463, 514), (463, 510), (463, 506), (469, 502), (469, 498), (469, 496), (469, 493), (469, 488), (469, 486), (469, 481), (469, 476), (474, 475), (474, 471), (474, 470), (474, 465), (474, 463), (474, 462), (474, 458), (474, 454), (474, 452), (474, 449), (474, 447), (474, 444), (474, 444), (478, 439), (478, 438), (478, 436), (478, 434), (478, 430), (478, 428), (478, 427), (478, 425), (478, 422), (478, 419), (478, 417), (478, 416), (478, 412), (478, 411), (478, 409), (478, 406), (478, 405), (478, 403), (481, 402), (481, 400), (481, 397), (481, 397), (481, 392), (481, 391), (481, 389), (481, 387), (481, 385), (481, 383), (481, 382), (481, 380), (481, 378), (481, 377), (481, 374), (481, 372), (481, 371), (481, 369), (486, 368), (486, 366), (486, 362), (486, 360), (486, 358), (486, 357), (486, 355), (486, 353), (486, 352), (486, 349), (486, 348), (486, 346), (486, 344), (486, 343), (486, 341), (486, 339), (490, 339), (490, 337), (490, 335), (490, 334), (490, 332), (490, 329), (490, 329), (490, 327), (490, 324), (490, 323), (490, 321), (490, 320), (490, 318), (490, 315), (490, 314), (493, 314), (493, 312), (493, 311), (493, 309), (493, 307), (493, 306), (493, 303), (493, 301), (493, 300), (493, 298), (493, 297), (493, 294), (493, 292), (493, 292), (493, 289), (493, 287), (493, 286), (493, 284), (493, 283), (498, 283)]"188 },189 {190 "step": 12,191 "screenshot": "1181506905418460_12.png",192 "action": "CLICK",193 "info": [194 [195 593,196 929197 ],198 [199 593,200 929201 ]202 ],203 "ps": "[(593, 929)]"204 },205 {206 "step": 13,207 "screenshot": "1181506905418460_13.png",208 "action": "CLICK",209 "info": [210 [211 424,212 341213 ],214 [215 424,216 341217 ]218 ],219 "ps": "[(424, 341)]"220 },221 {222 "step": 14,223 "screenshot": "1181506905418460_14.png",224 "action": "CLICK",225 "info": [226 [227 568,228 67229 ],230 [231 568,232 67233 ]234 ],235 "ps": "[(568, 67)]"236 },237 {238 "step": 15,239 "screenshot": "1181506905418460_15.png",240 "action": "CLICK",241 "info": [242 [243 783,244 61245 ],246 [247 783,248 61249 ]250 ],251 "ps": "[(783, 61)]"252 },253 {254 "step": 16,255 "screenshot": "1181506905418460_16.png",256 "action": "TEXT",257 "info": "vanilla extract",258 "ps": ""259 },260 {261 "step": 17,262 "screenshot": "1181506905418460_17.png",263 "action": "CLICK",264 "info": [265 [266 909,267 900268 ],269 [270 909,271 900272 ]273 ],274 "ps": "[(909, 900)]"275 },276 {277 "step": 18,278 "screenshot": "1181506905418460_18.png",279 "action": "CLICK",280 "info": [281 [282 649,283 492284 ],285 [286 649,287 492288 ]289 ],290 "ps": "[(649, 492)]"291 },292 {293 "step": 19,294 "screenshot": "1181506905418460_19.png",295 "action": "SCROLL",296 "info": [297 [298 605,299 796300 ],301 [302 722,303 364304 ]305 ],306 "ps": "[(605, 796), (605, 794), (605, 792), (605, 792), (605, 789), (605, 788), (605, 786), (605, 784), (605, 783), (605, 780), (605, 778), (605, 777), (605, 775), (605, 774), (605, 769), (605, 767), (610, 766), (610, 764), (610, 761), (610, 759), (610, 757), (612, 755), (612, 753), (612, 750), (612, 748), (612, 747), (617, 744), (617, 741), (617, 739), (617, 737), (617, 733), (622, 731), (622, 730), (622, 728), (622, 722), (622, 720), (624, 718), (624, 717), (624, 715), (624, 712), (624, 709), (629, 707), (629, 704), (629, 702), (629, 698), (629, 696), (629, 695), (629, 693), (634, 691), (634, 689), (634, 685), (634, 684), (634, 682), (637, 678), (637, 677), (637, 674), (637, 671), (637, 667), (641, 666), (641, 661), (641, 659), (641, 658), (641, 654), (641, 652), (646, 650), (646, 649), (646, 645), (646, 642), (646, 642), (649, 637), (649, 636), (649, 634), (649, 631), (649, 628), (649, 626), (653, 625), (653, 623), (653, 621), (653, 617), (653, 615), (653, 614), (659, 612), (659, 611), (659, 609), (659, 607), (659, 605), (659, 603), (659, 601), (659, 600), (661, 596), (661, 592), (661, 588), (666, 584), (666, 581), (666, 580), (666, 578), (666, 577), (666, 575), (666, 572), (666, 572), (671, 569), (671, 567), (671, 566), (671, 564), (671, 563), (671, 561), (671, 558), (675, 558), (675, 557), (675, 555), (675, 553), (675, 552), (675, 549), (675, 549), (675, 546), (675, 544), (675, 543), (675, 541), (675, 540), (675, 537), (678, 537), (678, 535), (678, 534), (678, 532), (678, 530), (678, 529), (683, 529), (683, 527), (683, 526), (683, 523), (683, 521), (683, 520), (683, 518), (683, 516), (683, 514), (683, 512), (683, 511), (686, 509), (686, 502), (690, 497), (690, 495), (690, 492), (690, 486), (690, 484), (690, 483), (690, 481), (695, 479), (695, 477), (695, 475), (695, 474), (695, 472), (695, 470), (695, 469), (698, 467), (698, 464), (698, 463), (698, 461), (698, 460), (698, 458), (698, 455), (698, 454), (698, 452), (698, 450), (702, 449), (702, 447), (702, 446), (702, 444), (702, 441), (702, 440), (707, 438), (707, 436), (707, 435), (707, 432), (707, 432), (707, 429), (707, 427), (707, 426), (707, 424), (707, 423), (710, 420), (710, 418), (710, 417), (710, 415), (710, 413), (710, 412), (710, 409), (710, 409), (714, 409), (714, 406), (714, 404), (714, 403), (714, 401), (714, 399), (714, 397), (714, 395), (714, 394), (714, 392), (714, 390), (714, 389), (717, 389), (717, 387), (717, 384), (717, 383), (717, 381), (717, 380), (717, 378), (717, 376), (722, 376), (722, 374), (722, 372), (722, 370), (722, 369), (722, 367), (722, 366), (722, 364)]"307 },308 {309 "step": 20,310 "screenshot": "1181506905418460_20.png",311 "action": "CLICK",312 "info": [313 [314 330,315 659316 ],317 [318 330,319 659320 ]321 ],322 "ps": "[(330, 659)]"323 },324 {325 "step": 21,326 "screenshot": "1181506905418460_21.png",327 "action": "COMPLETE",328 "info": "",329 "ps": ""330 }331 ]332}