OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0598611082940599",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2400,8 "w": 1080,9 "device_name": "Medium Phone"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Opera",15 "Things",16 "Ebay"17 ],18 "meta_task": "Find a recipe for {}, create a shopping list of main ingredients, and add to the cart.",19 "task": "Find a recipe for Chocolate chip cookies, create a shopping list of main ingredients, and add to the cart.",20 "instruction": "Using Opera, search for a recipe for Chocolate chip cookies. Then, organize a shopping list of the main ingredients in Things. Finally, add these ingredients to your cart on Ebay."21 },22 "step_length": 23,23 "steps": [24 {25 "step": 0,26 "screenshot": "0598611082940599_0.png",27 "action": "CLICK",28 "info": [29 [30 632,31 80832 ],33 [34 632,35 80836 ]37 ],38 "ps": "[(632, 808)]"39 },40 {41 "step": 1,42 "screenshot": "0598611082940599_1.png",43 "action": "CLICK",44 "info": "KEY_HOME",45 "ps": ""46 },47 {48 "step": 2,49 "screenshot": "0598611082940599_2.png",50 "action": "CLICK",51 "info": [52 [53 176,54 22955 ],56 [57 176,58 22959 ]60 ],61 "ps": "[(176, 229)]"62 },63 {64 "step": 3,65 "screenshot": "0598611082940599_3.png",66 "action": "CLICK",67 "info": [68 [69 855,70 91871 ],72 [73 855,74 91875 ]76 ],77 "ps": "[(855, 918)]"78 },79 {80 "step": 4,81 "screenshot": "0598611082940599_4.png",82 "action": "CLICK",83 "info": [84 [85 352,86 46487 ],88 [89 352,90 46491 ]92 ],93 "ps": "[(352, 464)]"94 },95 {96 "step": 5,97 "screenshot": "0598611082940599_5.png",98 "action": "TEXT",99 "info": "Chocolate chip cookies: 150g salted butter, softened 80g light brown muscovado sugar, 80g granulated sugar, 2 tsp vanilla extract, 1 large egg, 225g plain flour, 1/2 tsp bicarbonate of soda, 1/4 tsp salt, 200g plain chocolate chips or chunks",100 "ps": ""101 },102 {103 "step": 6,104 "screenshot": "0598611082940599_6.png",105 "action": "CLICK",106 "info": [107 [108 451,109 457110 ],111 [112 451,113 457114 ]115 ],116 "ps": "[(451, 457)]"117 },118 {119 "step": 7,120 "screenshot": "0598611082940599_7.png",121 "action": "CLICK",122 "info": "KEY_HOME",123 "ps": ""124 },125 {126 "step": 8,127 "screenshot": "0598611082940599_8.png",128 "action": "CLICK",129 "info": [130 [131 376,132 368133 ],134 [135 376,136 368137 ]138 ],139 "ps": "[(376, 368)]"140 },141 {142 "step": 9,143 "screenshot": "0598611082940599_9.png",144 "action": "CLICK",145 "info": [146 [147 60,148 58149 ],150 [151 60,152 58153 ]154 ],155 "ps": "[(60, 58)]"156 },157 {158 "step": 10,159 "screenshot": "0598611082940599_10.png",160 "action": "CLICK",161 "info": [162 [163 556,164 46165 ],166 [167 556,168 46169 ]170 ],171 "ps": "[(556, 46)]"172 },173 {174 "step": 11,175 "screenshot": "0598611082940599_11.png",176 "action": "CLICK",177 "info": [178 [179 899,180 917181 ],182 [183 899,184 917185 ]186 ],187 "ps": "[(899, 917)]"188 },189 {190 "step": 12,191 "screenshot": "0598611082940599_12.png",192 "action": "CLICK",193 "info": [194 [195 653,196 212197 ],198 [199 653,200 212201 ]202 ],203 "ps": "[(653, 212)]"204 },205 {206 "step": 13,207 "screenshot": "0598611082940599_13.png",208 "action": "SCROLL",209 "info": [210 [211 296,212 858213 ],214 [215 354,216 430217 ]218 ],219 "ps": "[(296, 858), (296, 857), (296, 856), (296, 854), (296, 852), (296, 850), (296, 848), (296, 847), (296, 845), (296, 843), (296, 842), (296, 839), (296, 837), (296, 836), (296, 834), (296, 833), (296, 831), (296, 829), (296, 827), (299, 825), (299, 823), (299, 822), (299, 820), (299, 819), (299, 817), (300, 814), (300, 813), (300, 811), (300, 808), (300, 806), (306, 803), (306, 802), (306, 798), (306, 797), (306, 794), (311, 792), (311, 789), (311, 787), (311, 786), (312, 782), (312, 780), (312, 776), (312, 775), (318, 773), (318, 768), (318, 767), (318, 763), (323, 759), (323, 757), (323, 755), (323, 752), (327, 750), (327, 747), (327, 744), (327, 741), (330, 739), (330, 736), (330, 731), (330, 729), (330, 726), (330, 724), (330, 720), (335, 718), (335, 715), (335, 713), (335, 712), (339, 707), (339, 703), (339, 699), (339, 695), (339, 693), (339, 689), (342, 687), (342, 684), (342, 682), (342, 679), (347, 677), (347, 673), (347, 671), (347, 668), (347, 666), (347, 663), (347, 660), (347, 658), (347, 654), (352, 652), (352, 649), (352, 647), (352, 646), (352, 642), (352, 640), (352, 638), (352, 634), (354, 633), (354, 631), (354, 629), (354, 625), (354, 624), (354, 619), (354, 617), (354, 616), (359, 612), (359, 609), (359, 607), (359, 604), (359, 603), (359, 601), (359, 600), (359, 598), (364, 596), (364, 594), (364, 592), (364, 589), (364, 587), (364, 584), (364, 584), (364, 581), (364, 579), (364, 578), (364, 576), (364, 575), (364, 570), (364, 569), (364, 567), (364, 565), (364, 564), (364, 562), (364, 559), (364, 558), (364, 556), (364, 555), (364, 553), (364, 551), (364, 549), (364, 547), (364, 545), (364, 544), (364, 533), (364, 530), (364, 528), (364, 521), (364, 518), (364, 516), (364, 514), (364, 512), (364, 510), (364, 508), (364, 507), (362, 507), (362, 505), (362, 504), (362, 502), (362, 499), (362, 498), (362, 496), (362, 494), (362, 493), (357, 493), (357, 490), (357, 489), (357, 487), (357, 485), (357, 484), (357, 482), (357, 481), (357, 479), (354, 479), (354, 476), (354, 475), (354, 473), (354, 471), (354, 470), (354, 467), (354, 467), (354, 464), (354, 462), (354, 461), (354, 459), (354, 457), (354, 455), (354, 453), (354, 452), (354, 450), (354, 448), (354, 447), (354, 444), (354, 442), (354, 441), (354, 439), (354, 438), (354, 436), (354, 434), (354, 432), (354, 430)]"220 },221 {222 "step": 14,223 "screenshot": "0598611082940599_14.png",224 "action": "CLICK",225 "info": [226 [227 172,228 564229 ],230 [231 172,232 564233 ]234 ],235 "ps": "[(172, 564)]"236 },237 {238 "step": 15,239 "screenshot": "0598611082940599_15.png",240 "action": "CLICK",241 "info": [242 [243 705,244 264245 ],246 [247 705,248 264249 ]250 ],251 "ps": "[(705, 264)]"252 },253 {254 "step": 16,255 "screenshot": "0598611082940599_16.png",256 "action": "CLICK",257 "info": [258 [259 563,260 64261 ],262 [263 563,264 64265 ]266 ],267 "ps": "[(563, 64)]"268 },269 {270 "step": 17,271 "screenshot": "0598611082940599_17.png",272 "action": "CLICK",273 "info": [274 [275 790,276 58277 ],278 [279 790,280 58281 ]282 ],283 "ps": "[(790, 58)]"284 },285 {286 "step": 18,287 "screenshot": "0598611082940599_18.png",288 "action": "CLICK",289 "info": [290 [291 364,292 261293 ],294 [295 364,296 261297 ]298 ],299 "ps": "[(364, 261)]"300 },301 {302 "step": 19,303 "screenshot": "0598611082940599_19.png",304 "action": "CLICK",305 "info": [306 [307 673,308 226309 ],310 [311 673,312 226313 ]314 ],315 "ps": "[(673, 226)]"316 },317 {318 "step": 20,319 "screenshot": "0598611082940599_20.png",320 "action": "SCROLL",321 "info": [322 [323 539,324 818325 ],326 [327 634,328 420329 ]330 ],331 "ps": "[(539, 818), (541, 815), (541, 813), (541, 812), (541, 810), (541, 809), (547, 807), (547, 804), (547, 803), (547, 801), (547, 800), (547, 798), (549, 796), (549, 794), (549, 792), (549, 790), (549, 789), (549, 787), (549, 786), (549, 784), (549, 782), (549, 780), (549, 778), (549, 776), (549, 775), (549, 773), (549, 772), (549, 769), (549, 767), (549, 766), (549, 764), (549, 762), (549, 761), (549, 759), (549, 757), (549, 755), (549, 753), (549, 752), (554, 750), (554, 749), (554, 747), (554, 744), (554, 743), (554, 741), (554, 739), (554, 738), (554, 736), (554, 734), (554, 732), (554, 730), (554, 729), (554, 727), (554, 724), (559, 724), (559, 719), (559, 718), (559, 716), (559, 714), (559, 713), (559, 710), (559, 708), (559, 707), (559, 705), (559, 702), (559, 699), (562, 698), (562, 696), (562, 694), (562, 693), (562, 691), (562, 689), (566, 685), (566, 683), (566, 682), (566, 680), (566, 677), (566, 672), (566, 670), (566, 669), (566, 667), (566, 664), (566, 663), (571, 661), (571, 660), (571, 656), (571, 654), (571, 652), (571, 650), (571, 649), (571, 647), (571, 645), (571, 642), (571, 639), (574, 638), (574, 636), (574, 634), (574, 633), (574, 628), (574, 627), (574, 625), (574, 623), (574, 619), (578, 617), (578, 616), (578, 614), (578, 613), (578, 611), (578, 609), (578, 607), (583, 603), (583, 602), (583, 600), (583, 598), (583, 597), (583, 594), (583, 592), (583, 591), (586, 589), (586, 588), (586, 586), (586, 584), (586, 582), (586, 580), (586, 578), (590, 577), (590, 575), (590, 574), (590, 572), (590, 569), (590, 568), (590, 566), (590, 564), (590, 563), (595, 561), (595, 559), (595, 557), (595, 555), (595, 554), (598, 552), (598, 549), (598, 549), (598, 546), (598, 545), (598, 543), (598, 541), (598, 540), (598, 537), (598, 537), (602, 534), (602, 532), (602, 531), (602, 529), (602, 527), (605, 526), (605, 523), (605, 522), (605, 520), (605, 518), (605, 517), (605, 514), (605, 512), (605, 511), (605, 509), (605, 508), (610, 506), (610, 504), (610, 502), (610, 500), (610, 498), (610, 497), (610, 495), (610, 494), (614, 492), (614, 489), (614, 488), (614, 486), (614, 484), (614, 483), (614, 481), (614, 479), (614, 477), (614, 475), (617, 475), (617, 474), (617, 472), (617, 471), (617, 469), (617, 467), (622, 454), (622, 451), (622, 449), (626, 449), (629, 438), (629, 435), (629, 432), (629, 432), (629, 429), (629, 428), (634, 426), (634, 424), (634, 423), (634, 420)]"332 },333 {334 "step": 21,335 "screenshot": "0598611082940599_21.png",336 "action": "CLICK",337 "info": [338 [339 449,340 707341 ],342 [343 449,344 707345 ]346 ],347 "ps": "[(449, 707)]"348 },349 {350 "step": 22,351 "screenshot": "0598611082940599_22.png",352 "action": "COMPLETE",353 "info": "",354 "ps": ""355 }356 ]357}