aps/super_glue

1---2annotations_creators:3- expert-generated4language_creators:5- other6language:7- en8license:9- other10multilinguality:11- monolingual12size_categories:13- 10K<n<100K14source_datasets:15- extended|other16task_categories:17- text-classification18- token-classification19- question-answering20task_ids:21- natural-language-inference22- word-sense-disambiguation23- coreference-resolution24- extractive-qa25paperswithcode_id: superglue26pretty_name: SuperGLUE27tags:28- superglue29- NLU30- natural language understanding31dataset_info:32- config_name: axb33  features:34  - name: sentence135    dtype: string36  - name: sentence237    dtype: string38  - name: idx39    dtype: int3240  - name: label41    dtype:42      class_label:43        names:44          '0': entailment45          '1': not_entailment46  splits:47  - name: test48    num_bytes: 23769449    num_examples: 110450  download_size: 8092451  dataset_size: 23769452- config_name: axg53  features:54  - name: premise55    dtype: string56  - name: hypothesis57    dtype: string58  - name: idx59    dtype: int3260  - name: label61    dtype:62      class_label:63        names:64          '0': entailment65          '1': not_entailment66  splits:67  - name: test68    num_bytes: 5334869    num_examples: 35670  download_size: 1428471  dataset_size: 5334872- config_name: boolq73  features:74  - name: question75    dtype: string76  - name: passage77    dtype: string78  - name: idx79    dtype: int3280  - name: label81    dtype:82      class_label:83        names:84        - 'False'85        - 'True'86  splits:87  - name: train88    num_bytes: 617330389    num_examples: 942790  - name: validation91    num_bytes: 211645292    num_examples: 327093  - name: test94    num_bytes: 210595995    num_examples: 324596  download_size: 647023197  dataset_size: 1039571498- config_name: cb99  features:100  - name: premise101    dtype: string102  - name: hypothesis103    dtype: string104  - name: idx105    dtype: int32106  - name: label107    dtype:108      class_label:109        names:110          '0': entailment111          '1': contradiction112          '2': neutral113  splits:114  - name: train115    num_bytes: 87050116    num_examples: 250117  - name: validation118    num_bytes: 21851119    num_examples: 56120  - name: test121    num_bytes: 93492122    num_examples: 250123  download_size: 139513124  dataset_size: 202393125- config_name: copa126  features:127  - name: premise128    dtype: string129  - name: choice1130    dtype: string131  - name: choice2132    dtype: string133  - name: question134    dtype: string135  - name: idx136    dtype: int32137  - name: label138    dtype:139      class_label:140        names:141          '0': choice1142          '1': choice2143  splits:144  - name: train145    num_bytes: 49233146    num_examples: 400147  - name: validation148    num_bytes: 12479149    num_examples: 100150  - name: test151    num_bytes: 59846152    num_examples: 500153  download_size: 86141154  dataset_size: 121558155- config_name: multirc156  features:157  - name: paragraph158    dtype: string159  - name: question160    dtype: string161  - name: answer162    dtype: string163  - name: idx164    struct:165    - name: paragraph166      dtype: int32167    - name: question168      dtype: int32169    - name: answer170      dtype: int32171  - name: label172    dtype:173      class_label:174        names:175        - 'False'176        - 'True'177  splits:178  - name: train179    num_bytes: 46182913180    num_examples: 27243181  - name: validation182    num_bytes: 7753452183    num_examples: 4848184  - name: test185    num_bytes: 14985531186    num_examples: 9693187  download_size: 2595545188  dataset_size: 68921896189- config_name: record190  features:191  - name: passage192    dtype: string193  - name: query194    dtype: string195  - name: entities196    sequence: string197  - name: entity_spans198    sequence:199    - name: text200      dtype: string201    - name: start202      dtype: int32203    - name: end204      dtype: int32205  - name: answers206    sequence: string207  - name: idx208    struct:209    - name: passage210      dtype: int32211    - name: query212      dtype: int32213  splits:214  - name: train215    num_bytes: 178757646216    num_examples: 100730217  - name: validation218    num_bytes: 17432944219    num_examples: 10000220  - name: test221    num_bytes: 17154943222    num_examples: 10000223  download_size: 91157911224  dataset_size: 213345533225- config_name: rte226  features:227  - name: premise228    dtype: string229  - name: hypothesis230    dtype: string231  - name: idx232    dtype: int32233  - name: label234    dtype:235      class_label:236        names:237          '0': entailment238          '1': not_entailment239  splits:240  - name: train241    num_bytes: 847177242    num_examples: 2490243  - name: validation244    num_bytes: 90716245    num_examples: 277246  - name: test247    num_bytes: 973916248    num_examples: 3000249  download_size: 1277939250  dataset_size: 1911809251- config_name: wic252  features:253  - name: word254    dtype: string255  - name: sentence1256    dtype: string257  - name: sentence2258    dtype: string259  - name: start1260    dtype: int32261  - name: start2262    dtype: int32263  - name: end1264    dtype: int32265  - name: end2266    dtype: int32267  - name: idx268    dtype: int32269  - name: label270    dtype:271      class_label:272        names:273        - 'False'274        - 'True'275  splits:276  - name: train277    num_bytes: 658381278    num_examples: 5428279  - name: validation280    num_bytes: 81811281    num_examples: 638282  - name: test283    num_bytes: 178831284    num_examples: 1400285  download_size: 597226286  dataset_size: 919023287- config_name: wsc288  features:289  - name: text290    dtype: string291  - name: span1_index292    dtype: int32293  - name: span2_index294    dtype: int32295  - name: span1_text296    dtype: string297  - name: span2_text298    dtype: string299  - name: idx300    dtype: int32301  - name: label302    dtype:303      class_label:304        names:305        - 'False'306        - 'True'307  splits:308  - name: train309    num_bytes: 89311310    num_examples: 554311  - name: validation312    num_bytes: 21521313    num_examples: 104314  - name: test315    num_bytes: 31408316    num_examples: 146317  download_size: 49592318  dataset_size: 142240319- config_name: wsc.fixed320  features:321  - name: text322    dtype: string323  - name: span1_index324    dtype: int32325  - name: span2_index326    dtype: int32327  - name: span1_text328    dtype: string329  - name: span2_text330    dtype: string331  - name: idx332    dtype: int32333  - name: label334    dtype:335      class_label:336        names:337        - 'False'338        - 'True'339  splits:340  - name: train341    num_bytes: 89311342    num_examples: 554343  - name: validation344    num_bytes: 21521345    num_examples: 104346  - name: test347    num_bytes: 31404348    num_examples: 146349  download_size: 49706350  dataset_size: 142236351configs:352- config_name: axb353  data_files:354  - split: test355    path: axb/test-*356- config_name: axg357  data_files:358  - split: test359    path: axg/test-*360- config_name: boolq361  data_files:362  - split: train363    path: boolq/train-*364  - split: validation365    path: boolq/validation-*366  - split: test367    path: boolq/test-*368- config_name: cb369  data_files:370  - split: train371    path: cb/train-*372  - split: validation373    path: cb/validation-*374  - split: test375    path: cb/test-*376- config_name: copa377  data_files:378  - split: train379    path: copa/train-*380  - split: validation381    path: copa/validation-*382  - split: test383    path: copa/test-*384- config_name: multirc385  data_files:386  - split: train387    path: multirc/train-*388  - split: validation389    path: multirc/validation-*390  - split: test391    path: multirc/test-*392- config_name: record393  data_files:394  - split: train395    path: record/train-*396  - split: validation397    path: record/validation-*398  - split: test399    path: record/test-*400- config_name: rte401  data_files:402  - split: train403    path: rte/train-*404  - split: validation405    path: rte/validation-*406  - split: test407    path: rte/test-*408- config_name: wic409  data_files:410  - split: train411    path: wic/train-*412  - split: validation413    path: wic/validation-*414  - split: test415    path: wic/test-*416- config_name: wsc417  data_files:418  - split: train419    path: wsc/train-*420  - split: validation421    path: wsc/validation-*422  - split: test423    path: wsc/test-*424- config_name: wsc.fixed425  data_files:426  - split: train427    path: wsc.fixed/train-*428  - split: validation429    path: wsc.fixed/validation-*430  - split: test431    path: wsc.fixed/test-*432---433 434# Dataset Card for "super_glue"435 436## Table of Contents437- [Dataset Description](#dataset-description)438  - [Dataset Summary](#dataset-summary)439  - [Supported Tasks and Leaderboards](#supported-tasks-and-leaderboards)440  - [Languages](#languages)441- [Dataset Structure](#dataset-structure)442  - [Data Instances](#data-instances)443  - [Data Fields](#data-fields)444  - [Data Splits](#data-splits)445- [Dataset Creation](#dataset-creation)446  - [Curation Rationale](#curation-rationale)447  - [Source Data](#source-data)448  - [Annotations](#annotations)449  - [Personal and Sensitive Information](#personal-and-sensitive-information)450- [Considerations for Using the Data](#considerations-for-using-the-data)451  - [Social Impact of Dataset](#social-impact-of-dataset)452  - [Discussion of Biases](#discussion-of-biases)453  - [Other Known Limitations](#other-known-limitations)454- [Additional Information](#additional-information)455  - [Dataset Curators](#dataset-curators)456  - [Licensing Information](#licensing-information)457  - [Citation Information](#citation-information)458  - [Contributions](#contributions)459 460## Dataset Description461 462- **Homepage:** https://super.gluebenchmark.com/463- **Repository:** [More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)464- **Paper:** https://arxiv.org/abs/1905.00537465- **Point of Contact:** [More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)466- **Size of downloaded dataset files:** 58.36 MB467- **Size of the generated dataset:** 249.57 MB468- **Total amount of disk used:** 307.94 MB469 470### Dataset Summary471 472SuperGLUE (https://super.gluebenchmark.com/) is a new benchmark styled after473GLUE with a new set of more difficult language understanding tasks, improved474resources, and a new public leaderboard.475 476### Supported Tasks and Leaderboards477 478[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)479 480### Languages481 482[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)483 484## Dataset Structure485 486### Data Instances487 488#### axb489 490- **Size of downloaded dataset files:** 0.03 MB491- **Size of the generated dataset:** 0.24 MB492- **Total amount of disk used:** 0.27 MB493 494An example of 'test' looks as follows.495```496 497```498 499#### axg500 501- **Size of downloaded dataset files:** 0.01 MB502- **Size of the generated dataset:** 0.05 MB503- **Total amount of disk used:** 0.06 MB504 505An example of 'test' looks as follows.506```507 508```509 510#### boolq511 512- **Size of downloaded dataset files:** 4.12 MB513- **Size of the generated dataset:** 10.40 MB514- **Total amount of disk used:** 14.52 MB515 516An example of 'train' looks as follows.517```518 519```520 521#### cb522 523- **Size of downloaded dataset files:** 0.07 MB524- **Size of the generated dataset:** 0.20 MB525- **Total amount of disk used:** 0.28 MB526 527An example of 'train' looks as follows.528```529 530```531 532#### copa533 534- **Size of downloaded dataset files:** 0.04 MB535- **Size of the generated dataset:** 0.13 MB536- **Total amount of disk used:** 0.17 MB537 538An example of 'train' looks as follows.539```540 541```542 543### Data Fields544 545The data fields are the same among all splits.546 547#### axb548- `sentence1`: a `string` feature.549- `sentence2`: a `string` feature.550- `idx`: a `int32` feature.551- `label`: a classification label, with possible values including `entailment` (0), `not_entailment` (1).552 553#### axg554- `premise`: a `string` feature.555- `hypothesis`: a `string` feature.556- `idx`: a `int32` feature.557- `label`: a classification label, with possible values including `entailment` (0), `not_entailment` (1).558 559#### boolq560- `question`: a `string` feature.561- `passage`: a `string` feature.562- `idx`: a `int32` feature.563- `label`: a classification label, with possible values including `False` (0), `True` (1).564 565#### cb566- `premise`: a `string` feature.567- `hypothesis`: a `string` feature.568- `idx`: a `int32` feature.569- `label`: a classification label, with possible values including `entailment` (0), `contradiction` (1), `neutral` (2).570 571#### copa572- `premise`: a `string` feature.573- `choice1`: a `string` feature.574- `choice2`: a `string` feature.575- `question`: a `string` feature.576- `idx`: a `int32` feature.577- `label`: a classification label, with possible values including `choice1` (0), `choice2` (1).578 579### Data Splits580 581#### axb582 583|   |test|584|---|---:|585|axb|1104|586 587#### axg588 589|   |test|590|---|---:|591|axg| 356|592 593#### boolq594 595|     |train|validation|test|596|-----|----:|---------:|---:|597|boolq| 9427|      3270|3245|598 599#### cb600 601|   |train|validation|test|602|---|----:|---------:|---:|603|cb |  250|        56| 250|604 605#### copa606 607|    |train|validation|test|608|----|----:|---------:|---:|609|copa|  400|       100| 500|610 611## Dataset Creation612 613### Curation Rationale614 615[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)616 617### Source Data618 619#### Initial Data Collection and Normalization620 621[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)622 623#### Who are the source language producers?624 625[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)626 627### Annotations628 629#### Annotation process630 631[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)632 633#### Who are the annotators?634 635[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)636 637### Personal and Sensitive Information638 639[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)640 641## Considerations for Using the Data642 643### Social Impact of Dataset644 645[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)646 647### Discussion of Biases648 649[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)650 651### Other Known Limitations652 653[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)654 655## Additional Information656 657### Dataset Curators658 659[More Information Needed](https://github.com/huggingface/datasets/blob/master/CONTRIBUTING.md#how-to-contribute-to-the-dataset-cards)660 661### Licensing Information662 663The primary SuperGLUE tasks are built on and derived from existing datasets. We refer users to the original licenses accompanying each dataset, but it is our understanding that these licenses allow for their use and redistribution in a research context.664 665### Citation Information666 667If you use SuperGLUE, please cite all the datasets you use in any papers that come out of your work. In addition, we encourage you to use the following BibTeX citation for SuperGLUE itself:668```669@article{wang2019superglue,670  title={Super{GLUE}: A Stickier Benchmark for General-Purpose Language Understanding Systems},671  author={Alex Wang and Yada Pruksachatkun and Nikita Nangia and Amanpreet Singh and Julian Michael and Felix Hill and Omer Levy and Samuel R. Bowman},672  journal={arXiv preprint 1905.00537},673  year={2019}674}675@inproceedings{clark2019boolq,676  title={{B}ool{Q}: Exploring the Surprising Difficulty of Natural Yes/No Questions},677  author={Clark, Christopher and Lee, Kenton and Chang, Ming-Wei and Kwiatkowski, Tom and Collins, Michael and Toutanova, Kristina},678  booktitle={Proceedings of NAACL-HLT 2019},679  year={2019}680}681@inproceedings{demarneffe:cb,682  title={{The CommitmentBank}: Investigating projection in naturally occurring discourse},683  author={De Marneffe, Marie-Catherine and Simons, Mandy and Tonhauser, Judith},684  note={To appear in proceedings of Sinn und Bedeutung 23. Data can be found at https://github.com/mcdm/CommitmentBank/},685  year={2019}686}687@inproceedings{roemmele2011choice,688  title={Choice of plausible alternatives: An evaluation of commonsense causal reasoning},689  author={Roemmele, Melissa and Bejan, Cosmin Adrian and Gordon, Andrew S.},690  booktitle={2011 AAAI Spring Symposium Series},691  year={2011}692}693@inproceedings{khashabi2018looking,694  title={Looking beyond the surface: A challenge set for reading comprehension over multiple sentences},695  author={Khashabi, Daniel and Chaturvedi, Snigdha and Roth, Michael and Upadhyay, Shyam and Roth, Dan},696  booktitle={Proceedings of the 2018 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long Papers)},697  pages={252--262},698  year={2018}699}700@article{zhang2018record,701  title={{ReCoRD}: Bridging the Gap between Human and Machine Commonsense Reading Comprehension},702  author={Sheng Zhang and Xiaodong Liu and Jingjing Liu and Jianfeng Gao and Kevin Duh and Benjamin Van Durme},703  journal={arXiv preprint 1810.12885},704  year={2018}705}706@incollection{dagan2006pascal,707  title={The {PASCAL} recognising textual entailment challenge},708  author={Dagan, Ido and Glickman, Oren and Magnini, Bernardo},709  booktitle={Machine learning challenges. evaluating predictive uncertainty, visual object classification, and recognising tectual entailment},710  pages={177--190},711  year={2006},712  publisher={Springer}713}714@article{bar2006second,715  title={The second {PASCAL} recognising textual entailment challenge},716  author={Bar Haim, Roy and Dagan, Ido and Dolan, Bill and Ferro, Lisa and Giampiccolo, Danilo and Magnini, Bernardo and Szpektor, Idan},717  year={2006}718}719@inproceedings{giampiccolo2007third,720  title={The third {PASCAL} recognizing textual entailment challenge},721  author={Giampiccolo, Danilo and Magnini, Bernardo and Dagan, Ido and Dolan, Bill},722  booktitle={Proceedings of the ACL-PASCAL workshop on textual entailment and paraphrasing},723  pages={1--9},724  year={2007},725  organization={Association for Computational Linguistics},726}727@article{bentivogli2009fifth,728  title={The Fifth {PASCAL} Recognizing Textual Entailment Challenge},729  author={Bentivogli, Luisa and Dagan, Ido and Dang, Hoa Trang and Giampiccolo, Danilo and Magnini, Bernardo},730  booktitle={TAC},731  year={2009}732}733@inproceedings{pilehvar2018wic,734  title={{WiC}: The Word-in-Context Dataset for Evaluating Context-Sensitive Meaning Representations},735  author={Pilehvar, Mohammad Taher and Camacho-Collados, Jose},736  booktitle={Proceedings of NAACL-HLT},737  year={2019}738}739@inproceedings{rudinger2018winogender,740  title={Gender Bias in Coreference Resolution},741  author={Rudinger, Rachel  and  Naradowsky, Jason  and  Leonard, Brian  and  {Van Durme}, Benjamin},742  booktitle={Proceedings of NAACL-HLT},743  year={2018}744}745@inproceedings{poliak2018dnc,746  title={Collecting Diverse Natural Language Inference Problems for Sentence Representation Evaluation},747  author={Poliak, Adam and Haldar, Aparajita and Rudinger, Rachel and Hu, J. Edward and Pavlick, Ellie and White, Aaron Steven and {Van Durme}, Benjamin},748  booktitle={Proceedings of EMNLP},749  year={2018}750}751@inproceedings{levesque2011winograd,752  title={The {W}inograd schema challenge},753  author={Levesque, Hector J and Davis, Ernest and Morgenstern, Leora},754  booktitle={{AAAI} Spring Symposium: Logical Formalizations of Commonsense Reasoning},755  volume={46},756  pages={47},757  year={2011}758}759```760 761### Contributions762 763Thanks to [@thomwolf](https://github.com/thomwolf), [@lewtun](https://github.com/lewtun), [@patrickvonplaten](https://github.com/patrickvonplaten) for adding this dataset.