OpenGVLab/GUI-Odyssey
Dataset Card for GUI Odyssey News⭐️ A new and improved version of the GUIOdyssey dataset has been released! 🎉🎉 👉 Please use the latest version and refer to the updated README for the most up-to-date information. We highly recommend using the new version for all training and evaluation! Repository: https://github.com/OpenGVLab/GUI-Odyssey Latest Version of Dataset: hflqf88888/GUIOdyssey Paper: https://arxiv.org/pdf/2406.08451 Introduction… See the full description on the dataset page: https://huggingface.co/datasets/OpenGVLab/GUI-Odyssey.
264.1k
1{2 "episode_id": "0508497834585831",3 "device_info": {4 "product": "sdk_gphone64_x86_64",5 "release_version": "14",6 "sdk_version": "34",7 "h": 2992,8 "w": 1344,9 "device_name": "Pixel 8 Pro"10 },11 "task_info": {12 "category": "Multi_Apps",13 "app": [14 "Duckduckgo Private Browser",15 "Microsoft Word",16 "Picturethis"17 ],18 "meta_task": "Find a photograph of a/an {}, identify it using a plant identifier app, and if the identification result is correct, record the image along with the plant name in a document.",19 "task": "Find a photograph of a/an Anemone, identify it using a plant identifier app, and if the identification result is correct, record the image along with the plant name in a document.",20 "instruction": "Locate a photograph of an Anemone using Duckduckgo Private Browser, then utilize Picturethis to identify the plant. If the identification is accurate, document the image and plant name in Microsoft Word."21 },22 "step_length": 31,23 "steps": [24 {25 "step": 0,26 "screenshot": "0508497834585831_0.png",27 "action": "CLICK",28 "info": [29 [30 183,31 26132 ],33 [34 183,35 26136 ]37 ],38 "ps": "[(183, 261)]"39 },40 {41 "step": 1,42 "screenshot": "0508497834585831_1.png",43 "action": "CLICK",44 "info": [45 [46 406,47 7048 ],49 [50 406,51 7052 ]53 ],54 "ps": "[(406, 70)]"55 },56 {57 "step": 2,58 "screenshot": "0508497834585831_2.png",59 "action": "TEXT",60 "info": "Anemone",61 "ps": ""62 },63 {64 "step": 3,65 "screenshot": "0508497834585831_3.png",66 "action": "CLICK",67 "info": [68 [69 951,70 91371 ],72 [73 951,74 91375 ]76 ],77 "ps": "[(951, 913)]"78 },79 {80 "step": 4,81 "screenshot": "0508497834585831_4.png",82 "action": "CLICK",83 "info": [84 [85 134,86 22087 ],88 [89 134,90 22091 ]92 ],93 "ps": "[(134, 220)]"94 },95 {96 "step": 5,97 "screenshot": "0508497834585831_5.png",98 "action": "CLICK",99 "info": [100 [101 351,102 693103 ],104 [105 351,106 693107 ]108 ],109 "ps": "[(351, 693)]"110 },111 {112 "step": 6,113 "screenshot": "0508497834585831_6.png",114 "action": "LONG_PRESS",115 "info": [116 [117 509,118 541119 ],120 [121 509,122 541123 ]124 ],125 "ps": "[(509, 541)]"126 },127 {128 "step": 7,129 "screenshot": "0508497834585831_7.png",130 "action": "CLICK",131 "info": [132 [133 362,134 523135 ],136 [137 362,138 523139 ]140 ],141 "ps": "[(362, 523)]"142 },143 {144 "step": 8,145 "screenshot": "0508497834585831_8.png",146 "action": "CLICK",147 "info": [148 [149 473,150 546151 ],152 [153 473,154 546155 ]156 ],157 "ps": "[(473, 546)]"158 },159 {160 "step": 9,161 "screenshot": "0508497834585831_9.png",162 "action": "CLICK",163 "info": [164 [165 393,166 523167 ],168 [169 393,170 523171 ]172 ],173 "ps": "[(393, 523)]"174 },175 {176 "step": 10,177 "screenshot": "0508497834585831_10.png",178 "action": "CLICK",179 "info": "KEY_HOME",180 "ps": ""181 },182 {183 "step": 11,184 "screenshot": "0508497834585831_11.png",185 "action": "CLICK",186 "info": [187 [188 382,189 399190 ],191 [192 382,193 399194 ]195 ],196 "ps": "[(382, 399)]"197 },198 {199 "step": 12,200 "screenshot": "0508497834585831_12.png",201 "action": "CLICK",202 "info": [203 [204 906,205 61206 ],207 [208 906,209 61210 ]211 ],212 "ps": "[(906, 61)]"213 },214 {215 "step": 13,216 "screenshot": "0508497834585831_13.png",217 "action": "CLICK",218 "info": [219 [220 502,221 913222 ],223 [224 502,225 913226 ]227 ],228 "ps": "[(502, 913)]"229 },230 {231 "step": 14,232 "screenshot": "0508497834585831_14.png",233 "action": "CLICK",234 "info": [235 [236 281,237 880238 ],239 [240 281,241 880242 ]243 ],244 "ps": "[(281, 880)]"245 },246 {247 "step": 15,248 "screenshot": "0508497834585831_15.png",249 "action": "CLICK",250 "info": [251 [252 348,253 751254 ],255 [256 348,257 751258 ]259 ],260 "ps": "[(348, 751)]"261 },262 {263 "step": 16,264 "screenshot": "0508497834585831_16.png",265 "action": "CLICK",266 "info": [267 [268 458,269 870270 ],271 [272 458,273 870274 ]275 ],276 "ps": "[(458, 870)]"277 },278 {279 "step": 17,280 "screenshot": "0508497834585831_17.png",281 "action": "CLICK",282 "info": "KEY_HOME",283 "ps": ""284 },285 {286 "step": 18,287 "screenshot": "0508497834585831_18.png",288 "action": "CLICK",289 "info": [290 [291 569,292 538293 ],294 [295 569,296 538297 ]298 ],299 "ps": "[(569, 538)]"300 },301 {302 "step": 19,303 "screenshot": "0508497834585831_19.png",304 "action": "CLICK",305 "info": [306 [307 940,308 88309 ],310 [311 940,312 88313 ]314 ],315 "ps": "[(940, 88)]"316 },317 {318 "step": 20,319 "screenshot": "0508497834585831_20.png",320 "action": "CLICK",321 "info": [322 [323 226,324 302325 ],326 [327 226,328 302329 ]330 ],331 "ps": "[(226, 302)]"332 },333 {334 "step": 21,335 "screenshot": "0508497834585831_21.png",336 "action": "TEXT",337 "info": "Anemone",338 "ps": ""339 },340 {341 "step": 22,342 "screenshot": "0508497834585831_22.png",343 "action": "SCROLL",344 "info": [345 [346 697,347 643348 ],349 [350 43,351 637352 ]353 ],354 "ps": "[(697, 643), (694, 643), (692, 643), (690, 643), (688, 643), (685, 643), (683, 643), (681, 643), (679, 643), (677, 643), (674, 643), (672, 643), (670, 643), (668, 643), (665, 643), (663, 643), (661, 643), (659, 643), (656, 643), (654, 643), (652, 643), (650, 643), (648, 643), (645, 643), (643, 643), (641, 643), (639, 643), (636, 643), (634, 643), (632, 643), (630, 643), (627, 643), (625, 643), (623, 643), (621, 643), (619, 643), (616, 643), (614, 643), (612, 643), (610, 643), (607, 643), (605, 643), (603, 643), (601, 643), (598, 643), (596, 644), (594, 644), (592, 644), (590, 644), (587, 644), (585, 644), (583, 644), (581, 644), (578, 644), (576, 644), (572, 644), (569, 644), (567, 644), (565, 644), (563, 645), (560, 645), (558, 645), (556, 645), (554, 645), (552, 645), (549, 645), (547, 645), (545, 645), (543, 645), (540, 645), (538, 645), (536, 645), (534, 645), (531, 645), (529, 646), (527, 646), (525, 646), (523, 646), (520, 646), (518, 646), (516, 646), (514, 646), (511, 646), (509, 646), (507, 646), (505, 646), (502, 646), (500, 646), (498, 646), (496, 646), (494, 646), (491, 646), (489, 646), (487, 646), (485, 646), (482, 646), (480, 646), (478, 646), (476, 646), (473, 646), (471, 646), (469, 646), (469, 647), (467, 647), (465, 647), (462, 647), (460, 647), (458, 647), (456, 647), (453, 647), (451, 647), (449, 647), (447, 647), (444, 647), (442, 647), (440, 647), (438, 647), (435, 647), (433, 647), (431, 647), (429, 647), (427, 647), (424, 647), (422, 647), (420, 647), (418, 647), (415, 647), (413, 647), (411, 647), (409, 647), (406, 647), (404, 647), (402, 647), (400, 647), (398, 647), (395, 647), (393, 647), (391, 647), (389, 647), (386, 647), (384, 647), (382, 647), (380, 647), (377, 647), (375, 647), (373, 647), (371, 647), (369, 647), (366, 647), (364, 647), (362, 647), (360, 647), (357, 647), (355, 647), (353, 647), (353, 646), (351, 646), (348, 646), (346, 646), (344, 646), (342, 646), (340, 646), (337, 646), (335, 646), (335, 645), (333, 645), (331, 645), (328, 645), (326, 645), (324, 645), (322, 645), (319, 645), (317, 645), (315, 645), (313, 645), (313, 644), (308, 644), (306, 644), (304, 644), (302, 644), (299, 644), (297, 644), (295, 644), (293, 644), (293, 643), (290, 643), (288, 643), (286, 643), (284, 643), (281, 643), (279, 643), (277, 643), (275, 643), (273, 643), (270, 643), (268, 643), (266, 643), (264, 643), (261, 643), (259, 643), (257, 643), (255, 643), (252, 643), (250, 643), (248, 643), (246, 643), (244, 643), (241, 643), (239, 643), (237, 643), (232, 643), (230, 643), (228, 643), (226, 643), (223, 643), (223, 642), (221, 642), (219, 642), (217, 642), (215, 642), (212, 642), (210, 642), (208, 642), (206, 642), (203, 642), (201, 642), (199, 642), (197, 642), (194, 642), (192, 641), (190, 641), (188, 641), (186, 641), (183, 641), (181, 641), (179, 641), (177, 641), (174, 641), (172, 641), (170, 640), (168, 640), (165, 640), (163, 640), (161, 640), (159, 640), (156, 640), (154, 640), (152, 640), (150, 640), (150, 639), (148, 639), (145, 639), (143, 639), (141, 639), (139, 639), (136, 639), (134, 639), (132, 639), (130, 639), (127, 639), (125, 639), (123, 638), (121, 638), (119, 638), (116, 638), (114, 638), (112, 638), (110, 638), (107, 638), (105, 638), (103, 638), (101, 638), (98, 638), (96, 638), (94, 638), (92, 638), (90, 638), (87, 638), (85, 638), (83, 638), (81, 638), (78, 638), (76, 638), (74, 638), (72, 638), (69, 638), (67, 638), (65, 638), (63, 638), (61, 638), (61, 637), (58, 637), (56, 637), (54, 637), (52, 637), (49, 637), (47, 637), (45, 637), (43, 637)]"355 },356 {357 "step": 23,358 "screenshot": "0508497834585831_23.png",359 "action": "SCROLL",360 "info": [361 [362 710,363 642364 ],365 [366 132,367 638368 ]369 ],370 "ps": "[(710, 642), (708, 642), (706, 642), (703, 642), (701, 642), (699, 642), (697, 642), (694, 642), (692, 642), (690, 642), (688, 642), (685, 642), (683, 642), (681, 642), (677, 642), (674, 642), (672, 642), (670, 642), (668, 642), (663, 642), (661, 642), (659, 642), (656, 642), (654, 642), (652, 642), (650, 642), (648, 642), (645, 642), (643, 642), (641, 642), (639, 642), (634, 642), (632, 642), (630, 642), (627, 642), (625, 642), (623, 642), (621, 642), (619, 642), (616, 642), (614, 642), (612, 642), (610, 642), (607, 642), (605, 642), (603, 642), (601, 642), (598, 642), (596, 642), (594, 642), (590, 642), (587, 642), (585, 642), (583, 642), (581, 642), (578, 642), (576, 642), (574, 642), (572, 642), (569, 642), (567, 642), (563, 642), (560, 642), (558, 642), (556, 642), (554, 642), (552, 642), (549, 642), (547, 642), (545, 642), (543, 642), (540, 642), (538, 642), (536, 642), (534, 642), (531, 642), (529, 642), (527, 642), (525, 642), (523, 642), (520, 642), (518, 642), (516, 642), (514, 642), (511, 642), (509, 642), (507, 642), (505, 642), (502, 642), (500, 642), (498, 642), (496, 642), (494, 642), (489, 642), (487, 643), (485, 643), (482, 643), (480, 643), (478, 643), (476, 643), (473, 643), (471, 643), (469, 643), (467, 643), (465, 643), (462, 643), (460, 643), (458, 643), (456, 643), (453, 643), (451, 643), (449, 643), (447, 643), (444, 643), (442, 643), (440, 643), (438, 643), (435, 643), (433, 643), (431, 643), (429, 643), (427, 643), (424, 643), (422, 643), (420, 643), (418, 643), (415, 643), (413, 643), (411, 643), (409, 643), (406, 643), (404, 643), (402, 643), (386, 643), (384, 643), (382, 643), (380, 643), (377, 643), (375, 643), (355, 643), (353, 643), (351, 643), (348, 643), (346, 643), (344, 643), (342, 643), (324, 643), (322, 643), (319, 643), (317, 643), (315, 643), (313, 643), (310, 643), (308, 643), (306, 643), (304, 643), (302, 643), (299, 643), (297, 643), (295, 643), (293, 643), (290, 643), (288, 643), (286, 643), (284, 643), (281, 643), (279, 643), (277, 643), (275, 643), (273, 642), (270, 642), (266, 642), (264, 642), (261, 642), (259, 642), (257, 642), (255, 642), (252, 642), (248, 642), (246, 642), (241, 642), (239, 642), (237, 642), (235, 642), (232, 642), (230, 642), (228, 641), (226, 641), (223, 641), (221, 641), (219, 641), (217, 641), (215, 641), (212, 641), (210, 641), (208, 641), (206, 641), (203, 641), (201, 641), (199, 641), (199, 640), (197, 640), (194, 640), (192, 640), (190, 640), (188, 640), (186, 640), (183, 640), (181, 640), (179, 640), (177, 640), (174, 640), (172, 640), (170, 640), (168, 640), (165, 640), (165, 639), (163, 639), (161, 639), (159, 639), (156, 639), (154, 639), (152, 639), (150, 639), (148, 639), (145, 639), (143, 639), (141, 639), (139, 639), (136, 639), (134, 639), (132, 638)]"371 },372 {373 "step": 24,374 "screenshot": "0508497834585831_24.png",375 "action": "SCROLL",376 "info": [377 [378 799,379 642380 ],381 [382 165,383 630384 ]385 ],386 "ps": "[(799, 642), (797, 642), (795, 642), (793, 642), (790, 642), (788, 642), (786, 642), (784, 642), (781, 642), (779, 642), (777, 642), (775, 642), (773, 642), (770, 642), (768, 642), (766, 642), (764, 642), (761, 642), (759, 642), (757, 642), (755, 642), (752, 642), (750, 642), (748, 642), (744, 642), (741, 642), (739, 642), (737, 642), (735, 642), (732, 642), (730, 642), (728, 642), (726, 642), (723, 642), (721, 641), (719, 641), (717, 641), (715, 641), (712, 641), (710, 641), (708, 641), (706, 641), (703, 641), (701, 641), (699, 640), (697, 640), (694, 640), (692, 640), (690, 640), (688, 640), (685, 640), (683, 640), (681, 640), (677, 640), (674, 640), (670, 640), (668, 639), (665, 639), (663, 639), (661, 639), (659, 639), (656, 639), (654, 639), (652, 639), (650, 639), (648, 639), (645, 639), (643, 639), (641, 639), (639, 639), (636, 639), (632, 639), (627, 638), (625, 638), (623, 638), (621, 638), (619, 638), (616, 638), (614, 638), (612, 638), (607, 638), (605, 638), (603, 638), (601, 638), (596, 638), (592, 638), (590, 638), (587, 638), (585, 638), (583, 638), (581, 638), (578, 638), (576, 638), (574, 638), (572, 638), (569, 638), (567, 638), (565, 638), (560, 638), (558, 638), (556, 638), (552, 638), (549, 638), (547, 638), (545, 638), (543, 638), (540, 638), (538, 638), (536, 638), (534, 638), (531, 638), (529, 638), (527, 638), (525, 638), (520, 638), (518, 638), (516, 638), (514, 638), (511, 638), (509, 637), (505, 637), (502, 637), (500, 637), (498, 637), (496, 637), (494, 637), (491, 637), (489, 637), (487, 637), (485, 637), (482, 637), (478, 637), (476, 637), (473, 637), (471, 637), (469, 637), (467, 637), (467, 636), (462, 636), (460, 636), (458, 636), (456, 636), (453, 636), (451, 636), (449, 636), (447, 635), (444, 635), (442, 635), (440, 635), (438, 635), (435, 635), (433, 635), (429, 635), (427, 635), (424, 635), (422, 635), (418, 635), (415, 635), (411, 634), (409, 634), (406, 634), (404, 634), (402, 634), (400, 634), (398, 634), (393, 634), (391, 633), (389, 633), (386, 633), (384, 633), (382, 633), (380, 633), (377, 633), (375, 633), (373, 633), (371, 633), (369, 633), (366, 633), (364, 633), (362, 633), (360, 633), (357, 633), (355, 633), (353, 633), (351, 633), (348, 633), (346, 633), (346, 632), (342, 632), (340, 632), (337, 632), (335, 632), (333, 632), (331, 632), (328, 632), (326, 632), (322, 632), (319, 632), (317, 632), (313, 632), (310, 632), (308, 632), (306, 632), (304, 632), (302, 632), (299, 632), (297, 632), (293, 632), (290, 632), (288, 632), (286, 632), (284, 632), (281, 632), (279, 632), (277, 632), (275, 632), (273, 632), (270, 632), (268, 632), (266, 632), (264, 632), (261, 632), (259, 632), (257, 632), (255, 632), (252, 632), (250, 632), (248, 631), (246, 631), (244, 631), (241, 631), (239, 631), (237, 631), (235, 631), (232, 631), (230, 631), (228, 631), (226, 631), (223, 631), (221, 631), (219, 631), (217, 631), (215, 631), (212, 631), (210, 631), (208, 631), (206, 630), (203, 630), (201, 630), (199, 630), (197, 630), (194, 630), (192, 630), (190, 630), (188, 630), (186, 630), (183, 630), (181, 630), (179, 630), (177, 630), (174, 630), (172, 630), (170, 630), (168, 630), (165, 630)]"387 },388 {389 "step": 25,390 "screenshot": "0508497834585831_25.png",391 "action": "CLICK",392 "info": [393 [394 498,395 640396 ],397 [398 498,399 640400 ]401 ],402 "ps": "[(498, 640)]"403 },404 {405 "step": 26,406 "screenshot": "0508497834585831_26.png",407 "action": "CLICK",408 "info": [409 [410 536,411 565412 ],413 [414 536,415 565416 ]417 ],418 "ps": "[(536, 565)]"419 },420 {421 "step": 27,422 "screenshot": "0508497834585831_27.png",423 "action": "CLICK",424 "info": [425 [426 208,427 190428 ],429 [430 208,431 190432 ]433 ],434 "ps": "[(208, 190)]"435 },436 {437 "step": 28,438 "screenshot": "0508497834585831_28.png",439 "action": "CLICK",440 "info": [441 [442 880,443 938444 ],445 [446 880,447 938448 ]449 ],450 "ps": "[(880, 938)]"451 },452 {453 "step": 29,454 "screenshot": "0508497834585831_29.png",455 "action": "CLICK",456 "info": [457 [458 860,459 950460 ],461 [462 860,463 950464 ]465 ],466 "ps": "[(860, 950)]"467 },468 {469 "step": 30,470 "screenshot": "0508497834585831_30.png",471 "action": "COMPLETE",472 "info": "",473 "ps": "7 LONG 8 download failed 9 LONG"474 }475 ]476}