rfr2003/GeoBenchLLM

1---2language:3- en4license: other5pretty_name: Geo Benchmark6task_categories:7- text-generation8configs:9- config_name: GKMC10  data_files:11  - split: test12    path: GKMC/test-*13- config_name: GeoQuery_place14  data_files:15  - split: train16    path: GeoQuery_place/train-*17  - split: validation18    path: GeoQuery_place/validation-*19  - split: test20    path: GeoQuery_place/test-*21- config_name: GeoQuery_regression22  data_files:23  - split: train24    path: GeoQuery_regression/train-*25  - split: validation26    path: GeoQuery_regression/validation-*27  - split: test28    path: GeoQuery_regression/test-*29- config_name: GeoQuestions1089_YN30  data_files:31  - split: test32    path: GeoQuestions1089_YN/test-*33- config_name: GeoQuestions1089_coord34  data_files:35  - split: test36    path: GeoQuestions1089_coord/test-*37- config_name: GeoQuestions1089_place38  data_files:39  - split: test40    path: GeoQuestions1089_place/test-*41- config_name: GeoQuestions1089_regression42  data_files:43  - split: test44    path: GeoQuestions1089_regression/test-*45- config_name: GeoSQA46  data_files:47  - split: train48    path: GeoSQA/train-*49  - split: validation50    path: GeoSQA/validation-*51  - split: test52    path: GeoSQA/test-*53- config_name: GridRoute54  data_files:55  - split: test56    path: GridRoute/test-*57- config_name: MsMarco58  data_files:59  - split: test60    path: MsMarco/test-*61  - split: train62    path: MsMarco/train-*63  - split: validation64    path: MsMarco/validation-*65- config_name: NY-POI66  data_files:67  - split: test68    path: NY-POI/test-*69- config_name: PPNL_multi70  data_files:71  - split: test72    path: PPNL_multi/test-*73  - split: train74    path: PPNL_multi/train-*75  - split: validation76    path: PPNL_multi/validation-*77- config_name: PPNL_single78  data_files:79  - split: test80    path: PPNL_single/test-*81  - split: train82    path: PPNL_single/train-*83  - split: validation84    path: PPNL_single/validation-*85- config_name: SpartUN86  data_files:87  - split: test88    path: SpartUN/test-*89  - split: train90    path: SpartUN/train-*91  - split: validation92    path: SpartUN/validation-*93- config_name: SpatialEvalLLM94  data_files:95  - split: test96    path: SpatialEvalLLM/test-*97- config_name: StepGame98  data_files:99  - split: train100    path: StepGame/train-*101  - split: validation102    path: StepGame/validation-*103  - split: test104    path: StepGame/test-*105- config_name: TourismQA106  data_files:107  - split: test108    path: TourismQA/test-*109  - split: train110    path: TourismQA/train-*111  - split: validation112    path: TourismQA/validation-*113dataset_info:114- config_name: GKMC115  features:116  - name: question_id117    dtype: int64118  - name: answer119    dtype: string120  - name: scenario121    dtype: string122  - name: question123    dtype: string124  - name: A125    dtype: string126  - name: B127    dtype: string128  - name: C129    dtype: string130  - name: D131    dtype: string132  splits:133  - name: test134    num_bytes: 1055828135    num_examples: 1600136  download_size: 510919137  dataset_size: 1055828138- config_name: GeoQuery_place139  features:140  - name: question141    dtype: string142  - name: answer143    list: string144  splits:145  - name: train146    num_bytes: 57875147    num_examples: 346148  - name: validation149    num_bytes: 4037150    num_examples: 33151  - name: test152    num_bytes: 27964153    num_examples: 184154  download_size: 30317155  dataset_size: 89876156- config_name: GeoQuery_regression157  features:158  - name: question159    dtype: string160  - name: answer161    list: float64162  splits:163  - name: train164    num_bytes: 12026165    num_examples: 182166  - name: validation167    num_bytes: 1017168    num_examples: 17169  - name: test170    num_bytes: 5966171    num_examples: 89172  download_size: 13105173  dataset_size: 19009174- config_name: GeoQuestions1089_YN175  features:176  - name: question_id177    dtype: int64178  - name: question179    dtype: string180  - name: answer181    list: bool182  - name: answer_type183    list: string184  splits:185  - name: test186    num_bytes: 12412187    num_examples: 181188  download_size: 7718189  dataset_size: 12412190- config_name: GeoQuestions1089_coord191  features:192  - name: question_id193    dtype: int64194  - name: question195    dtype: string196  - name: answer197    list:198      list: float64199  - name: answer_type200    list: string201  splits:202  - name: test203    num_bytes: 7042204    num_examples: 87205  download_size: 6242206  dataset_size: 7042207- config_name: GeoQuestions1089_place208  features:209  - name: question_id210    dtype: int64211  - name: question212    dtype: string213  - name: answer214    list: string215  - name: answer_type216    list: string217  splits:218  - name: test219    num_bytes: 4373368220    num_examples: 455221  download_size: 1896109222  dataset_size: 4373368223- config_name: GeoQuestions1089_regression224  features:225  - name: question_id226    dtype: int64227  - name: question228    dtype: string229  - name: answer230    list: float64231  - name: answer_type232    list: string233  splits:234  - name: test235    num_bytes: 20649236    num_examples: 231237  download_size: 10655238  dataset_size: 20649239- config_name: GeoSQA240  features:241  - name: question_id242    dtype: int64243  - name: scenario_id244    dtype: int64245  - name: answer246    dtype: string247  - name: annotation248    dtype: string249  - name: scenario250    dtype: string251  - name: question252    dtype: string253  - name: A254    dtype: string255  - name: B256    dtype: string257  - name: C258    dtype: string259  - name: D260    dtype: string261  splits:262  - name: train263    num_bytes: 2350343264    num_examples: 2644265  - name: validation266    num_bytes: 566689267    num_examples: 628268  - name: test269    num_bytes: 762135270    num_examples: 838271  download_size: 1327080272  dataset_size: 3679167273- config_name: GridRoute274  features:275  - name: matrix_size276    dtype: int64277  - name: start278    list: int64279  - name: end280    list:281      list: int64282  - name: obstacles_coords283    list:284      list: int64285  - name: path286    list:287      list: int64288  splits:289  - name: test290    num_bytes: 439500291    num_examples: 300292  download_size: 16947293  dataset_size: 439500294- config_name: MsMarco295  features:296  - name: question_id297    dtype: int64298  - name: question299    dtype: string300  - name: answer301    dtype: string302  - name: passages303    list:304    - name: is_selected305      dtype: int64306    - name: passage_text307      dtype: string308    - name: url309      dtype: string310  splits:311  - name: test312    num_bytes: 10860618313    num_examples: 2907314  - name: train315    num_bytes: 90739271316    num_examples: 23513317  - name: validation318    num_bytes: 16126312319    num_examples: 4149320  download_size: 58502647321  dataset_size: 117726201322- config_name: NY-POI323  features:324  - name: long-term_check-ins325    list:326      list: string327  - name: recent_check-ins328    list:329      list: string330  - name: candidates331    list:332      list: string333  - name: answer334    list: string335  splits:336  - name: test337    num_bytes: 9088765338    num_examples: 1347339  download_size: 3829714340  dataset_size: 9088765341- config_name: PPNL_multi342  features:343  - name: matrix_size344    dtype: int64345  - name: world_description346    dtype: string347  - name: world348    list:349      list: int64350  - name: obstacles_coords351    list:352      list: int64353  - name: start354    list: int64355  - name: end356    list:357      list: int64358  - name: n_goals359    dtype: int64360  - name: path361    list:362      list: int64363  - name: agent_as_a_point364    dtype: string365  - name: agent_has_direction366    dtype: string367  - name: distribution368    dtype: string369  splits:370  - name: test371    num_bytes: 80282702372    num_examples: 55440373  - name: train374    num_bytes: 76667038375    num_examples: 53440376  - name: validation377    num_bytes: 9587004378    num_examples: 6680379  download_size: 13201821380  dataset_size: 166536744381- config_name: PPNL_single382  features:383  - name: matrix_size384    dtype: int64385  - name: world_description386    dtype: string387  - name: world388    list:389      list: int64390  - name: obstacles_coords391    list:392      list: int64393  - name: start394    list: int64395  - name: end396    list:397      list: int64398  - name: n_goals399    dtype: int64400  - name: path401    list:402      list: int64403  - name: agent_as_a_point404    dtype: string405  - name: agent_has_direction406    dtype: string407  - name: distribution408    dtype: string409  splits:410  - name: test411    num_bytes: 15738553412    num_examples: 19044413  - name: train414    num_bytes: 12749254415    num_examples: 16032416  - name: validation417    num_bytes: 1594684418    num_examples: 2004419  download_size: 1341236420  dataset_size: 30082491421- config_name: SpartUN422  features:423  - name: scenario_id424    dtype: string425  - name: question_id426    dtype: string427  - name: scenario428    dtype: string429  - name: question430    dtype: string431  - name: candidates_answers432    list: string433  - name: answer434    list: string435  - name: type436    dtype: string437  - name: k_hop438    dtype: int64439  splits:440  - name: test441    num_bytes: 3597916442    num_examples: 5551443  - name: train444    num_bytes: 24431833445    num_examples: 37095446  - name: validation447    num_bytes: 3562581448    num_examples: 5600449  download_size: 3174385450  dataset_size: 31592330451- config_name: SpatialEvalLLM452  features:453  - name: question454    dtype: string455  - name: answer456    list: string457  - name: scenario458    dtype: string459  - name: struct_type460    dtype: string461  - name: size462    dtype: string463  - name: k_hop464    dtype: string465  - name: seed466    dtype: string467  - name: description_level468    dtype: string469  splits:470  - name: test471    num_bytes: 1100727472    num_examples: 1400473  download_size: 203297474  dataset_size: 1100727475- config_name: StepGame476  features:477  - name: scenario478    dtype: string479  - name: question480    dtype: string481  - name: answer482    dtype: string483  - name: k_hop484    dtype: int64485  - name: candidates_answers486    list: string487  splits:488  - name: train489    num_bytes: 15871676490    num_examples: 50000491  - name: validation492    num_bytes: 1584479493    num_examples: 5000494  - name: test495    num_bytes: 65697697496    num_examples: 100000497  download_size: 18189276498  dataset_size: 83153852499- config_name: TourismQA500  features:501  - name: question502    dtype: string503  - name: city504    struct:505    - name: coord506      list: float64507    - name: name508      dtype: string509  - name: tagged_locations510    list: string511  - name: tagged_locations_lat_long512    list:513      list: float64514  - name: answers_names515    list: string516  - name: answers_adresses517    list: string518  - name: answers_sum_reviews519    list: string520  - name: answers_reviews521    list:522      list: string523  - name: answers_lat_longs524    list:525      list: float64526  splits:527  - name: test528    num_bytes: 7590110529    num_examples: 2153530  - name: train531    num_bytes: 74777277532    num_examples: 19762533  - name: validation534    num_bytes: 7343485535    num_examples: 2109536  download_size: 45062284537  dataset_size: 89710872538---539 540# 🌍 GeoBenchLLM541 542<center>543    <img src="https://github.com/Rfr2003/GeoBenchLLM/blob/main/geobenchllm_logo.png?raw=true" alt="GeoBenchLLM: A comprehensive benchmark for probing LLM on geo-related tasks">544</center>545 546## Table of Contents547 548## Dataset Description549 550- **Homepage:** https://github.com/Rfr2003/GeoBenchLLM551- **Repository:** https://github.com/Rfr2003/GeoBenchLLM552- **Paper:** 553- **Point of Contact:** rodrigo.ferreira-rodrigues@utoulouse.fr554 555### Benchmark Summary556 557GeoBenchLLM aims to assess Large Language Models' (LLM) geographical abilities across a multitude of tasks. It is built from 12 datasets split across 8 differents tasks:558 559- Knowledge/**Coordinates Prediction** : [GeoQuestions1089](https://github.com/AI-team-UoA/GeoQuestions1089)560- Knowledge/**Yes|No questions**: [GeoQuestions1089](https://github.com/AI-team-UoA/GeoQuestions1089)561- Knowledge/**Regression questions**: [GeoQuestions1089](https://github.com/AI-team-UoA/GeoQuestions1089), [GeoQuery](https://www.cs.utexas.edu/~ml/nldata/geoquery.html)562- Knowledge/**Place Prediction**: [GeoQuestions1089](https://github.com/AI-team-UoA/GeoQuestions1089), [GeoQuery](https://www.cs.utexas.edu/~ml/nldata/geoquery.html), [Ms Marco](https://microsoft.github.io/msmarco/)563- Reasoning/**Scenario Complex QA**: [GeoSQA](http://ws.nju.edu.cn/gaokao/geosqa/1.0/), [GKMC](https://github.com/nju-websoft/Jeeves-GKMC)564- Reasoning/**Spatial Reasoning**: [SpartUN](https://github.com/HLR/SpaRTUN), [StepGame](https://github.com/ShiZhengyan/StepGame), [SpatialEvalLLM](https://github.com/runopti/SpatialEvalLLM)565- Application/**POI Recommendation**: [TourismQA](https://github.com/dair-iitd/TourismQA), [NY-QA](https://sites.google.com/site/yangdingqi/home/foursquare-dataset)566- Application/**Path Finding**: [GridRoute](https://github.com/LinChance/GridRoute), [PPNL](https://github.com/MohamedAghzal/llms-as-path-planners)567 568These datasets have been preprocessed in order to be easily accessible.569 570 571```python572import datasets573 574dataset = datasets.load_dataset("rfr2003/GeoBenchLLM", "GeoSQA")575```576 577### Supported Tasks and Leaderboards578 579The dataset is used for Text Generation.580 581### Languages582 583All datasets are in English (`en`).584 585## Dataset Structure586 587As this dataset contains very heterogenous tasks, almost every dataset as a different data structure. 588 589### Data Instances590 591Please report to the dataset viewer to see what an instance for each dataset looks like.592 593### Data Fields594 595We will give for each dataset the data fields. Note that fields highlighted by 🟦 are required to formulate the question and fields highlighted by 🟩 contain the answer to the question. Every other fields can either be used to perform some analytics or to formulate differents tasks on the same dataset.596 597- **GeoQuestions1089_coord**:598    - 🟦 `question`(`str`) : the question to be answered.599    - 🟩 `answer`(`List[float]`) : the coordinates of the answer. The first element of the list correspond to the latitude and the second to the longitude.600- **GeoQuestions1089_YN**:601    - 🟦 `question`(`str`) : the question to be answered.602    - 🟩 `answer`(`List[bool]`) : a list containing the boolean corresponding to the answer.603- **GeoQuestions1089_regression** and **GeoQuery_regression**:604    - 🟦 `question`(`str`) : the question to be answered.605    - 🟩 `answer`(`List[float]`) : a list containing the numbers to be predicted.606- **GeoQuestions1089_place** and **GeoQuery_place**:607    - 🟦 `question`(`str`) : the question to be answered.608    - 🟩 `answer`(`List[str]`) : a list containing the names of the places to be predicted.609- **Ms-Marco_place**:610    - 🟦 `question`(`str`) : the question to be answered.611    - 🟩 `answer`(`str`) : the answer to the question formulated by a human.612    - `question_id`(`int64`) : the id of the question from the original dataset.613    - `passages`(`List[dict]`) : a list of dicts. Each dict correspond to a passage and gives the following information:614      - `is_selected`(`int64`) : 1 if the passage was selected to write the answer, 0 otherwise.615      - `passage_text`(`str`) : the text of the passage.616      - `url`(`str`) : the url from where the passage was retrieved.617- **GeoSQA**:618    - 🟦 `annotation`(`str`) : the description of the image normally used to answer the question.619    - 🟦 `scenario`(`str`) : the scenario attached to the image providing context to the question.620    - 🟦 `question`(`str`) : the question to be answered.621    - 🟦 `A`(`str`) : one of the possibles answers to the question.622    - 🟦 `B`(`str`) : one of the possibles answers to the question.623    - 🟦 `C`(`str`) : one of the possibles answers to the question.624    - 🟦 `D`(`str`) : one of the possibles answers to the question.625    - 🟩 `answer`(`str`) : the letter corresponding to the right choice.626    - `question_id`(`int64`) : the id of the question from the original dataset.627    - `scenario_id`(`int64`) : the id of the scenario from the original dataset.628- **GKMC**:629    - 🟦 `scenario`(`str`) : the scenario providing context to the question.630    - 🟦 `question`(`str`) : the question to be answered.631    - 🟦 `A`(`str`) : one of the possibles answers to the question.632    - 🟦 `B`(`str`) : one of the possibles answers to the question.633    - 🟦 `C`(`str`) : one of the possibles answers to the question.634    - 🟦 `D`(`str`) : one of the possibles answers to the question.635    - 🟩 `answer`(`str`) : the letter corresponding to the right choice.636    - `question_id`(`int64`) : the id of the question from the original dataset.637- **SpatialEvalLLM**:638    - 🟦 `scenario`(`str`) : the scenario providing context to the question.639    - 🟦 `question`(`str`) : the question to be answered.640    - 🟩 `answer`(`List[str]`) : a list containing the names of the right objects to predict.641    - `struct_type`(`str`) : the geometric structure of the map.642    - `size`(`str`) : the size of the structure in number of tiles composing it.643    - `k_hop`(`str`) : the minimum reasoning steps required to answer the question.644    - `seed`(`str`) : the seed used to generate the question.645    - `description_level`(`str`) : if **global** then the entierity of the map is described. If **local**, only a portion of the map is described.646- **SpartUN**:647    - 🟦 `scenario`(`str`) : the scenario providing context to the question.648    - 🟦 `question`(`str`) : the question to be answered.649    - 🟦 `candidates_answers`(`List[str]`) : the candidates answers from which the model has to retrieve.650    - 🟩 `answer`(`List[str]`) : a list containing the right answers from the candidate list.651    - `question_id`(`str`) : the id of the question from the original dataset.652    - `scenario_id`(`str`) : the id of the scenario from the original dataset.653    - `type`(`str`) : **YN** from boolean questions, **FR** for Find Relation questions.654    - `k_hop`(`int64`) : the minimum reasoning steps required to answer the question.655- **StepGame**:656    - 🟦 `scenario`(`str`) : the scenario providing context to the question.657    - 🟦 `question`(`str`) : the question to be answered.658    - 🟦 `candidates_answers`(`List[str]`) : the candidates answers from which the model has to retrieve.659    - 🟩 `answer`(`List[str]`) : a list containing the right answers from the candidate list.660    - `k_hop`(`int64`) : the minimum reasoning steps required to answer the question.661- **TourismQA**:662    - 🟦 `question`(`str`) : the question to be answered.663    - 🟩 `answers_names`(`List[str]`) : a list containing the names of the POI to be recommended (answer expected).664    - `city`(`dict`) : a dict containing the following informations about the city where take place the question:665      - `coord`(`List[float]`) : the coordinates of the city. The first element of the list correspond to the latitude and the second to the longitude.666      - `name`(`str`) : the name of the city.667    - `tagged_locations`(`List[str]`) : the locations names retrieved from the question (not used for our description of the task).668    - `tagged_locations_lat_long`(`List[flaot]`) : the latitudes and longitudes of the locations retrieved from the question (not used for our description of the task).669    - `answers_adresses`(`List[str]`) : the postal adresses of each answer (not used for our description of the task).670    - `answers_reviews`(`List[List[str]]`) : for each POI, we have a list of reviews (not used for our description of the task).671    - `answers_sum_reviews`(`List[str]`) : a summarization of the reviews for each POI retrieved from ??? work (not used for our description of the task).672    - `answers_lat_longs`(`List[str]`) : the latitudes and longitudes of the answers (not used for our description of the task).673- **NY-POI**:674    - 🟦 `long-term_check-ins`(`List[List[str]]`) : a list of long-term check-ins from the same user. Each check-in is list composed in the order : POI id, POI category and time of visit in UTC.675    - 🟦 `recent_check-ins`(`List[List[str]]`) : a list of recent check-ins from an user. Each check-in is list composed in the order : POI id, POI category and time of visit in UTC.676    - 🟦 `candidates`(`List[List[str]]`) : a list of POI candidates containing the answer. For each POI, we have its id, its distance from the last visited POI in the recent check-ins list and its category.677    - 🟩 `answer`(`List[str]`) : the id of the POI corresponding to the answer.678- **GridRoute**:679    - 🟦 `matrix_size`(`int64`) : the size of the squared matrix.680    - 🟦 `start`(`List[int64]`) : the coordinates of the starting point. The first element of the list is the x coordinate and the second is the y one.681    - 🟦 `end`(`List[List[int64]]`) : a list of ending points that the has to reach. This dataset only has one end point per question.682    - 🟦 `obstacles_coords`(`List[List[int64]]`) : a list of coordinates corresponding to the obstacles that we have to avoid. For each point, the first element of the list is the x coordinate and the second is the y one.683    - 🟩 `path`(`List[List[int64]]`) : a list of coordinates corresponding to the optimal path. For each point, the first element of the list is the x coordinate and the second is the y one.684- **PPNL_single**:685    - 🟦 `matrix_size`(`int64`) : the size of the squared matrix.686    - 🟦 `start`(`List[int64]`) : the coordinates of the starting point. The first element of the list is the x coordinate and the second is the y one.687    - 🟦 `end`(`List[List[int64]]`) : a list of ending points that the has to reach. This dataset only has one end point per question.688    - 🟦 `obstacles_coords`(`List[List[int64]]`) : a list of coordinates corresponding to the obstacles that we have to avoid. For each point, the first element of the list is the x coordinate and the second is the y one.689    - 🟩 `path`(`List[List[int64]]`) : a list of coordinates corresponding to the optimal path. For each point, the first element of the list is the x coordinate and the second is the y one. If there is no path possible, this field is an empty list.690    - `world_description`(`str`) : a description of the world in natural language. Can be used to directly prompt the model.691    - `n_goals`(`int64`) : the number of end points to reach.692    - `agent_as_a_point`(`str`) : the solution path described as if the model is a point.693    - `agent_has_direction`(`str`) : the solution path described as directions.694    - `distribution`(`str`) : **iid** if the example has the same properties (matrix size, initial location/goal placements and number of obstacles), **ood** otherwise.695- **PPNL_multi**:696    - 🟦 `matrix_size`(`int64`) : the size of the squared matrix.697    - 🟦 `start`(`List[int64]`) : the coordinates of the starting point. The first element of the list is the x coordinate and the second is the y one.698    - 🟦 `end`(`List[List[int64]]`) : a list of ending points that the has to reach. Each question has at least 2 goals to reach.699    - 🟦 `obstacles_coords`(`List[List[int64]]`) : a list of coordinates corresponding to the obstacles that we have to avoid. For each point, the first element of the list is the x coordinate and the second is the y one.700    - 🟩 `path`(`List[List[int64]]`) : a list of coordinates corresponding to the optimal path. For each point, the first element of the list is the x coordinate and the second is the y one. If there is no path possible, this field is an empty list.701    - `world_description`(`str`) : a description of the world in natural language. Can be used to directly prompt the model.702    - `n_goals`(`int64`) : the number of end points to reach.703    - `agent_as_a_point`(`str`) : the solution path described as if the model is a point.704    - `agent_has_direction`(`str`) : the solution path described as directions.705    - `distribution`(`str`) : **iid** if the example has the same properties (matrix size,  initial location/goal placements and number of obstacles), **ood** otherwise.706 707 708### Data Splits709 710| Cogn. Level       | Tasks                  | Datasets                                 | Train                 | Dev                 | Test                      |711| --------------- | ---------------------- | ---------------------------------------- | --------------------- | ------------------- | ------------------------- |712| **Knowledge**   | Coordinates Prediction | GeoQuestions1089_coord                        | –                     | –                   | 87                        |713|                 | Yes/No questions       | GeoQuestions1089_YN                         | –                     | –                   | 181                       |714|                 | Regression             | GeoQuestions1089_regression<br>GeoQuery_regression             | –<br>182              | –<br>17             | 231<br>89                 |715|                 | Place prediction       | GeoQuestions1089_place<br>GeoQuery_place<br>MS-Marco_place | –<br>346<br>23 513    | –<br>33<br>4 149    | 455<br>184<br>2 907       |716| **──────────**  | **──────────**         | **──────────**                           | **──────────**        | **──────────**      | **──────────**            |717| **Reasoning**   | Scenario Complex QA    | GeoSQA<br>GKMC                           | 2 644<br>–                | 628<br>–              | 838<br>1 600            |718|                 | Spatial Reasoning      | SpatialEvalLLM<br>SpartUN<br>StepGame    | –<br>37 095<br>50 000 | –<br>5 600<br>5 000 | 1 400<br>5 551<br>100 000 |719| **──────────**  | **──────────**         | **──────────**                           | **──────────**        | **──────────**      | **──────────**            |720| **Application** | POI Recommendation     | TourismQA<br>NY-QA                       | 19 762<br>–           | 2 109<br>–          | 2 153<br>1 347            |721|                 | Path Finding           | GridRoute<br>PPNL_single<br>PPNL_multi  | –<br>16 032<br>53 440  | –<br>2 004<br>6 680 | 300<br>19 044<br>55 440    |722| **──────────**  | **──────────**         | **──────────**                           | **──────────**        | **──────────**      | **──────────**            |723| **Total**       | –                      | –                                        | **203 014**           | **26 220**          | **191 807**               |724 725 726 727## Dataset Creation728 729### Curation Rationale730 731[Needs More Information]732 733### Source Data734 735#### Initial Data Collection and Normalization736 737[Needs More Information]738 739#### Who are the source language producers?740 741[Needs More Information]742 743### Annotations744 745#### Annotation process746 747[Needs More Information]748 749#### Who are the annotators?750 751[Needs More Information]752 753### Personal and Sensitive Information754 755[Needs More Information]756 757## Considerations for Using the Data758 759### Social Impact of Dataset760 761[Needs More Information]762 763### Discussion of Biases764 765[Needs More Information]766 767### Other Known Limitations768 769[Needs More Information]770 771## Additional Information772 773### Dataset Curators774 775[Needs More Information]776 777### Licensing Information778 779[Needs More Information]780 781### Citation Information782 783Thanks for all the authors of the all the datasets. If you use this Benchmark, please cite their work too.784 785```Tex786@misc{huang2021retrieverreadermeetsscenariobasedmultiplechoice,787      title={When Retriever-Reader Meets Scenario-Based Multiple-Choice Questions}, 788      author={Zixian Huang and Ao Wu and Yulin Shen and Gong Cheng and Yuzhong Qu},789      year={2021},790      eprint={2108.13875},791      archivePrefix={arXiv},792      primaryClass={cs.CL},793      url={https://arxiv.org/abs/2108.13875}, 794}795 796@inproceedings{finegan-dollak-etal-2018-improving,797    title = "Improving Text-to-{SQL} Evaluation Methodology",798    author = "Finegan-Dollak, Catherine  and799      Kummerfeld, Jonathan K.  and800      Zhang, Li  and801      Ramanathan, Karthik  and802      Sadasivam, Sesh  and803      Zhang, Rui  and804      Radev, Dragomir",805    editor = "Gurevych, Iryna  and806      Miyao, Yusuke",807    booktitle = "Proceedings of the 56th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers)",808    month = jul,809    year = "2018",810    address = "Melbourne, Australia",811    publisher = "Association for Computational Linguistics",812    url = "https://aclanthology.org/P18-1033/",813    doi = "10.18653/v1/P18-1033",814    pages = "351--360",815}816 817@inproceedings{data-geography-original818  dataset   = {Geography, original},819  author    = {John M. Zelle and Raymond J. Mooney},820  title     = {Learning to Parse Database Queries Using Inductive Logic Programming},821  booktitle = {Proceedings of the Thirteenth National Conference on Artificial Intelligence - Volume 2},822  year      = {1996},823  pages     = {1050--1055},824  location  = {Portland, Oregon},825  url       = {http://dl.acm.org/citation.cfm?id=1864519.1864543},826}827 828@misc{huang2019geosqabenchmarkscenariobasedquestion,829      title={GeoSQA: A Benchmark for Scenario-based Question Answering in the Geography Domain at High School Level}, 830      author={Zixian Huang and Yulin Shen and Xiao Li and Yuang Wei and Gong Cheng and Lin Zhou and Xinyu Dai and Yuzhong Qu},831      year={2019},832      eprint={1908.07855},833      archivePrefix={arXiv},834      primaryClass={cs.CL},835      url={https://arxiv.org/abs/1908.07855}, 836}837 838@misc{li2025gridroutebenchmarkllmbasedroute,839      title={GridRoute: A Benchmark for LLM-Based Route Planning with Cardinal Movement in Grid Environments}, 840      author={Kechen Li and Yaotian Tao and Ximing Wen and Quanwei Sun and Zifei Gong and Chang Xu and Xizhe Zhang and Tianbo Ji},841      year={2025},842      eprint={2505.24306},843      archivePrefix={arXiv},844      primaryClass={cs.AI},845      url={https://arxiv.org/abs/2505.24306}, 846}847 848@article{DBLP:journals/corr/NguyenRSGTMD16,849  author    = {Tri Nguyen and850               Mir Rosenberg and851               Xia Song and852               Jianfeng Gao and853               Saurabh Tiwary and854               Rangan Majumder and855               Li Deng},856  title     = {{MS} {MARCO:} {A} Human Generated MAchine Reading COmprehension Dataset},857  journal   = {CoRR},858  volume    = {abs/1611.09268},859  year      = {2016},860  url       = {http://arxiv.org/abs/1611.09268},861  archivePrefix = {arXiv},862  eprint    = {1611.09268},863  timestamp = {Mon, 13 Aug 2018 16:49:03 +0200},864  biburl    = {https://dblp.org/rec/journals/corr/NguyenRSGTMD16.bib},865  bibsource = {dblp computer science bibliography, https://dblp.org}866}867 868@inbook{placequestions,869author = {Hamzei, Ehsan and Li, Haonan and Vasardani, Maria and Baldwin, Timothy and Winter, Stephan and Tomko, Martin},870year = {2020},871month = {01},872pages = {3-19},873title = {Place Questions and Human-Generated Answers: A Data Analysis Approach},874isbn = {978-3-030-14745-7},875doi = {10.1007/978-3-030-14745-7_1}876}877 878@inproceedings{aghzal2024can,879  title={Can Large Language Models be Good Path Planners? A Benchmark and Investigation on Spatial-temporal Reasoning},880  author={Aghzal, Mohamed and Plaku, Erion and Yao, Ziyu},881  booktitle={ICLR 2024 Workshop on Large Language Model (LLM) Agents},882  year={2024}883}884 885@inproceedings{mirzaee-kordjamshidi-2022-transfer,886    title = "Transfer Learning with Synthetic Corpora for Spatial Role Labeling and Reasoning",887    author = "Mirzaee, Roshanak  and888      Kordjamshidi, Parisa",889    booktitle = "Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing",890    month = dec,891    year = "2022",892    address = "Abu Dhabi, United Arab Emirates",893    publisher = "Association for Computational Linguistics",894    url = "https://aclanthology.org/2022.emnlp-main.413",895    pages = "6148--6165",896    abstract = "",897}898 899@article{yamada2023evaluating,900    title={Evaluating Spatial Understanding of Large Language Models},901    author={Yamada, Yutaro and Bao, Yihan and Lampinen, Andrew K and Kasai, Jungo and Yildirim, Ilker},902    journal={Transactions on Machine Learning Research},903    year={2024}904}905 906@inproceedings{10.1145/3459637.3482320,907    author = {Contractor, Danish and Shah, Krunal and Partap, Aditi and Singla, Parag and Mausam, Mausam},908    title = {Answering POI-recommendation Questions using Tourism Reviews},909    year = {2021},910    isbn = {9781450384469},911    publisher = {Association for Computing Machinery},912    address = {New York, NY, USA},913    url = {https://doi.org/10.1145/3459637.3482320},914    doi = {10.1145/3459637.3482320},915    booktitle = {Proceedings of the 30th ACM International Conference on Information \& Knowledge Management},916    pages = {281–291},917    numpages = {11},918    keywords = {large scale qa, poi-recommendation, question answering, real world task, tourism qa},919    location = {Virtual Event, Queensland, Australia},920    series = {CIKM '21}921}922 923 924@misc{li2024locationawaremodularbiencoder,925      title={Location Aware Modular Biencoder for Tourism Question Answering}, 926      author={Haonan Li and Martin Tomko and Timothy Baldwin},927      year={2024},928      eprint={2401.02187},929      archivePrefix={arXiv},930      primaryClass={cs.CL},931      url={https://arxiv.org/abs/2401.02187}, 932}933 934@inproceedings{10.1007/978-3-031-47243-5_15,935  title = {Benchmarking Geospatial Question Answering Engines Using the Dataset GeoQuestions1089},936  author = {Sergios-Anestis Kefalidis, Dharmen Punjani, Eleni Tsalapati, 937         Konstantinos Plas, Mariangela Pollali, Michail Mitsios, 938         Myrto Tsokanaridou, Manolis Koubarakis and Pierre Maret},939  booktitle = {The Semantic Web - {ISWC} 2023 - 22nd International Semantic Web Conference,940            Athens, Greece, November 6-10, 2023, Proceedings, Part {II}},941  year = {2023}942}943 944@inproceedings{stepGame2022shi,945    title={StepGame: A New Benchmark for Robust Multi-Hop Spatial Reasoning in Texts},946    author={Shi, Zhengxiang and Zhang, Qiang and Lipani, Aldo},947    volume={36},948    url={https://ojs.aaai.org/index.php/AAAI/article/view/21383},949    DOI={10.1609/aaai.v36i10.21383}, 950    booktitle={Proceedings of the AAAI Conference on Artificial Intelligence},951    year={2022},952    month={Jun.},953    pages={11321-11329}954}955 956@inproceedings{Yang_2022, series={SIGIR ’22},957   title={GETNext: Trajectory Flow Map Enhanced Transformer for Next POI Recommendation},958   url={http://dx.doi.org/10.1145/3477495.3531983},959   DOI={10.1145/3477495.3531983},960   booktitle={Proceedings of the 45th International ACM SIGIR Conference on Research and Development in Information Retrieval},961   publisher={ACM},962   author={Yang, Song and Liu, Jiamou and Zhao, Kaiqi},963   year={2022},964   month=jul, pages={1144–1153},965   collection={SIGIR ’22} 966}967   968@ARTICLE{6844862,969  author={Yang, Dingqi and Zhang, Daqing and Zheng, Vincent W. and Yu, Zhiyong},970  journal={IEEE Transactions on Systems, Man, and Cybernetics: Systems}, 971  title={Modeling User Activity Preference by Leveraging User Spatial Temporal Characteristics in LBSNs}, 972  year={2015},973  volume={45},974  number={1},975  pages={129-142},976  keywords={Tensile stress;Data models;Context modeling;Correlation;Hidden Markov models;Location based social networks;spatial;temporal;tensor factorization;user activity preference;Location based social networks;spatial;temporal;tensor factorization;user activity preference},977  doi={10.1109/TSMC.2014.2327053}978}979 980@inproceedings{10.1145/3539618.3591770,981    author = {Yan, Xiaodong and Song, Tengwei and Jiao, Yifeng and He, Jianshan and Wang, Jiaotuan and Li, Ruopeng and Chu, Wei},982    title = {Spatio-Temporal Hypergraph Learning for Next POI Recommendation},983    year = {2023},984    isbn = {9781450394086},985    publisher = {Association for Computing Machinery},986    address = {New York, NY, USA},987    url = {https://doi.org/10.1145/3539618.3591770},988    doi = {10.1145/3539618.3591770},989    booktitle = {Proceedings of the 46th International ACM SIGIR Conference on Research and Development in Information Retrieval},990    pages = {403–412},991    numpages = {10},992    keywords = {graph transformer, hypergraph, next poi recommendation},993    location = {Taipei, Taiwan},994    series = {SIGIR '23}995}996 997@INPROCEEDINGS{10605522,998  author={Feng, Shanshan and Lyu, Haoming and Li, Fan and Sun, Zhu and Chen, Caishun},999  booktitle={2024 IEEE Conference on Artificial Intelligence (CAI)}, 1000  title={Where to Move Next: Zero-shot Generalization of LLMs for Next POI Recommendation}, 1001  year={2024},1002  volume={},1003  number={},1004  pages={1530-1535},1005  keywords={Accuracy;Large language models;Computational modeling;Buildings;Chatbots;Cognition;Data models;LLMs;Next POI Recommendation;Zero-shot;Spatial-Temporal Data},1006  doi={10.1109/CAI59869.2024.00277}1007}1008 1009```1010 1011### Contributions1012 1013TO DO1014