mteb/CodeSearchNetRetrieval

1---2annotations_creators:3- derived4language:5- code6license: mit7multilinguality: monolingual8source_datasets:9- code-search-net/code_search_net10task_categories:11- text-retrieval12task_ids: []13dataset_info:14- config_name: go-corpus15  features:16  - name: id17    dtype: string18  - name: text19    dtype: string20  - name: title21    dtype: string22  splits:23  - name: test24    num_bytes: 68792325    num_examples: 100026  download_size: 30512627  dataset_size: 68792328- config_name: go-qrels29  features:30  - name: query-id31    dtype: string32  - name: corpus-id33    dtype: string34  - name: score35    dtype: int6436  splits:37  - name: test38    num_bytes: 13108439    num_examples: 100040  download_size: 3064241  dataset_size: 13108442- config_name: go-queries43  features:44  - name: id45    dtype: string46  - name: text47    dtype: string48  splits:49  - name: test50    num_bytes: 13610551    num_examples: 100052  download_size: 8201753  dataset_size: 13610554- config_name: java-corpus55  features:56  - name: id57    dtype: string58  - name: text59    dtype: string60  - name: title61    dtype: string62  splits:63  - name: test64    num_bytes: 59139865    num_examples: 100066  download_size: 18660667  dataset_size: 59139868- config_name: java-qrels69  features:70  - name: query-id71    dtype: string72  - name: corpus-id73    dtype: string74  - name: score75    dtype: int6476  splits:77  - name: test78    num_bytes: 17634179    num_examples: 100080  download_size: 3495381  dataset_size: 17634182- config_name: java-queries83  features:84  - name: id85    dtype: string86  - name: text87    dtype: string88  splits:89  - name: test90    num_bytes: 70195891    num_examples: 100092  download_size: 25548693  dataset_size: 70195894- config_name: javascript-corpus95  features:96  - name: id97    dtype: string98  - name: text99    dtype: string100  - name: title101    dtype: string102  splits:103  - name: test104    num_bytes: 1551805105    num_examples: 1000106  download_size: 571030107  dataset_size: 1551805108- config_name: javascript-qrels109  features:110  - name: query-id111    dtype: string112  - name: corpus-id113    dtype: string114  - name: score115    dtype: int64116  splits:117  - name: test118    num_bytes: 141801119    num_examples: 1000120  download_size: 39930121  dataset_size: 141801122- config_name: javascript-queries123  features:124  - name: id125    dtype: string126  - name: text127    dtype: string128  splits:129  - name: test130    num_bytes: 202696131    num_examples: 1000132  download_size: 112208133  dataset_size: 202696134- config_name: php-corpus135  features:136  - name: id137    dtype: string138  - name: text139    dtype: string140  - name: title141    dtype: string142  splits:143  - name: test144    num_bytes: 856887145    num_examples: 1000146  download_size: 333818147  dataset_size: 856887148- config_name: php-qrels149  features:150  - name: query-id151    dtype: string152  - name: corpus-id153    dtype: string154  - name: score155    dtype: int64156  splits:157  - name: test158    num_bytes: 151365159    num_examples: 1000160  download_size: 33419161  dataset_size: 151365162- config_name: php-queries163  features:164  - name: id165    dtype: string166  - name: text167    dtype: string168  splits:169  - name: test170    num_bytes: 173189171    num_examples: 1000172  download_size: 88243173  dataset_size: 173189174- config_name: python-corpus175  features:176  - name: id177    dtype: string178  - name: text179    dtype: string180  - name: title181    dtype: string182  splits:183  - name: test184    num_bytes: 1008096185    num_examples: 1000186  download_size: 397446187  dataset_size: 1008096188- config_name: python-qrels189  features:190  - name: query-id191    dtype: string192  - name: corpus-id193    dtype: string194  - name: score195    dtype: int64196  splits:197  - name: test198    num_bytes: 152118199    num_examples: 1000200  download_size: 32945201  dataset_size: 152118202- config_name: python-queries203  features:204  - name: id205    dtype: string206  - name: text207    dtype: string208  splits:209  - name: test210    num_bytes: 477722211    num_examples: 1000212  download_size: 232579213  dataset_size: 477722214- config_name: ruby-corpus215  features:216  - name: id217    dtype: string218  - name: text219    dtype: string220  - name: title221    dtype: string222  splits:223  - name: test224    num_bytes: 713048225    num_examples: 1000226  download_size: 300121227  dataset_size: 713048228- config_name: ruby-qrels229  features:230  - name: query-id231    dtype: string232  - name: corpus-id233    dtype: string234  - name: score235    dtype: int64236  splits:237  - name: test238    num_bytes: 142286239    num_examples: 1000240  download_size: 43267241  dataset_size: 142286242- config_name: ruby-queries243  features:244  - name: id245    dtype: string246  - name: text247    dtype: string248  splits:249  - name: test250    num_bytes: 324866251    num_examples: 1000252  download_size: 169887253  dataset_size: 324866254configs:255- config_name: go-corpus256  data_files:257  - split: test258    path: go-corpus/test-*259- config_name: go-qrels260  data_files:261  - split: test262    path: go-qrels/test-*263- config_name: go-queries264  data_files:265  - split: test266    path: go-queries/test-*267- config_name: java-corpus268  data_files:269  - split: test270    path: java-corpus/test-*271- config_name: java-qrels272  data_files:273  - split: test274    path: java-qrels/test-*275- config_name: java-queries276  data_files:277  - split: test278    path: java-queries/test-*279- config_name: javascript-corpus280  data_files:281  - split: test282    path: javascript-corpus/test-*283- config_name: javascript-qrels284  data_files:285  - split: test286    path: javascript-qrels/test-*287- config_name: javascript-queries288  data_files:289  - split: test290    path: javascript-queries/test-*291- config_name: php-corpus292  data_files:293  - split: test294    path: php-corpus/test-*295- config_name: php-qrels296  data_files:297  - split: test298    path: php-qrels/test-*299- config_name: php-queries300  data_files:301  - split: test302    path: php-queries/test-*303- config_name: python-corpus304  data_files:305  - split: test306    path: python-corpus/test-*307- config_name: python-qrels308  data_files:309  - split: test310    path: python-qrels/test-*311- config_name: python-queries312  data_files:313  - split: test314    path: python-queries/test-*315- config_name: ruby-corpus316  data_files:317  - split: test318    path: ruby-corpus/test-*319- config_name: ruby-qrels320  data_files:321  - split: test322    path: ruby-qrels/test-*323- config_name: ruby-queries324  data_files:325  - split: test326    path: ruby-queries/test-*327tags:328- mteb329- text330---331<!-- adapted from https://github.com/huggingface/huggingface_hub/blob/v0.30.2/src/huggingface_hub/templates/datasetcard_template.md -->332 333<div align="center" style="padding: 40px 20px; background-color: white; border-radius: 12px; box-shadow: 0 2px 10px rgba(0, 0, 0, 0.05); max-width: 600px; margin: 0 auto;">334  <h1 style="font-size: 3.5rem; color: #1a1a1a; margin: 0 0 20px 0; letter-spacing: 2px; font-weight: 700;">CodeSearchNetRetrieval</h1>335  <div style="font-size: 1.5rem; color: #4a4a4a; margin-bottom: 5px; font-weight: 300;">An <a href="https://github.com/embeddings-benchmark/mteb" style="color: #2c5282; font-weight: 600; text-decoration: none;" onmouseover="this.style.textDecoration='underline'" onmouseout="this.style.textDecoration='none'">MTEB</a> dataset</div>336  <div style="font-size: 0.9rem; color: #2c5282; margin-top: 10px;">Massive Text Embedding Benchmark</div>337</div>338 339The dataset is a collection of code snippets and their corresponding natural language queries. The task is to retrieve the most relevant code snippet for a given query.340 341|               |                                             |342|---------------|---------------------------------------------|343| Task category | t2t                              |344| Domains       | Programming, Written                               |345| Reference     | https://huggingface.co/datasets/code_search_net/ |346 347Source datasets:348- [code-search-net/code_search_net](https://huggingface.co/datasets/code-search-net/code_search_net)349 350 351## How to evaluate on this task352 353You can evaluate an embedding model on this dataset using the following code:354 355```python356import mteb357 358task = mteb.get_task("CodeSearchNetRetrieval")359evaluator = mteb.MTEB([task])360 361model = mteb.get_model(YOUR_MODEL)362evaluator.run(model)363```364 365<!-- Datasets want link to arxiv in readme to autolink dataset with paper -->366To learn more about how to run models on `mteb` task check out the [GitHub repository](https://github.com/embeddings-benchmark/mteb).367 368## Citation369 370If you use this dataset, please cite the dataset as well as [mteb](https://github.com/embeddings-benchmark/mteb), as this dataset likely includes additional processing as a part of the [MMTEB Contribution](https://github.com/embeddings-benchmark/mteb/tree/main/docs/mmteb).371 372```bibtex373 374@article{husain2019codesearchnet,375  author = {Husain, Hamel and Wu, Ho-Hsiang and Gazit, Tiferet and Allamanis, Miltiadis and Brockschmidt, Marc},376  journal = {arXiv preprint arXiv:1909.09436},377  title = {{CodeSearchNet} challenge: Evaluating the state of semantic code search},378  year = {2019},379}380 381 382@article{enevoldsen2025mmtebmassivemultilingualtext,383  title={MMTEB: Massive Multilingual Text Embedding Benchmark},384  author={Kenneth Enevoldsen and Isaac Chung and Imene Kerboua and Márton Kardos and Ashwin Mathur and David Stap and Jay Gala and Wissam Siblini and Dominik Krzemiński and Genta Indra Winata and Saba Sturua and Saiteja Utpala and Mathieu Ciancone and Marion Schaeffer and Gabriel Sequeira and Diganta Misra and Shreeya Dhakal and Jonathan Rystrøm and Roman Solomatin and Ömer Çağatan and Akash Kundu and Martin Bernstorff and Shitao Xiao and Akshita Sukhlecha and Bhavish Pahwa and Rafał Poświata and Kranthi Kiran GV and Shawon Ashraf and Daniel Auras and Björn Plüster and Jan Philipp Harries and Loïc Magne and Isabelle Mohr and Mariya Hendriksen and Dawei Zhu and Hippolyte Gisserot-Boukhlef and Tom Aarsen and Jan Kostkan and Konrad Wojtasik and Taemin Lee and Marek Šuppa and Crystina Zhang and Roberta Rocca and Mohammed Hamdy and Andrianos Michail and John Yang and Manuel Faysse and Aleksei Vatolin and Nandan Thakur and Manan Dey and Dipam Vasani and Pranjal Chitale and Simone Tedeschi and Nguyen Tai and Artem Snegirev and Michael Günther and Mengzhou Xia and Weijia Shi and Xing Han Lù and Jordan Clive and Gayatri Krishnakumar and Anna Maksimova and Silvan Wehrli and Maria Tikhonova and Henil Panchal and Aleksandr Abramov and Malte Ostendorff and Zheng Liu and Simon Clematide and Lester James Miranda and Alena Fenogenova and Guangyu Song and Ruqiya Bin Safi and Wen-Ding Li and Alessia Borghini and Federico Cassano and Hongjin Su and Jimmy Lin and Howard Yen and Lasse Hansen and Sara Hooker and Chenghao Xiao and Vaibhav Adlakha and Orion Weller and Siva Reddy and Niklas Muennighoff},385  publisher = {arXiv},386  journal={arXiv preprint arXiv:2502.13595},387  year={2025},388  url={https://arxiv.org/abs/2502.13595},389  doi = {10.48550/arXiv.2502.13595},390}391 392@article{muennighoff2022mteb,393  author = {Muennighoff, Niklas and Tazi, Nouamane and Magne, Loïc and Reimers, Nils},394  title = {MTEB: Massive Text Embedding Benchmark},395  publisher = {arXiv},396  journal={arXiv preprint arXiv:2210.07316},397  year = {2022}398  url = {https://arxiv.org/abs/2210.07316},399  doi = {10.48550/ARXIV.2210.07316},400}401```402 403# Dataset Statistics404<details>405  <summary> Dataset Statistics</summary>406 407The following code contains the descriptive statistics from the task. These can also be obtained using:408 409```python410import mteb411 412task = mteb.get_task("CodeSearchNetRetrieval")413 414desc_stats = task.metadata.descriptive_stats415```416 417```json418{419    "test": {420        "num_samples": 12000,421        "number_of_characters": 6496327,422        "documents_text_statistics": {423            "total_text_length": 4552253,424            "min_text_length": 69,425            "average_text_length": 758.7088333333334,426            "max_text_length": 334374,427            "unique_texts": 6000428        },429        "documents_image_statistics": null,430        "queries_text_statistics": {431            "total_text_length": 1944074,432            "min_text_length": 2,433            "average_text_length": 324.01233333333334,434            "max_text_length": 17533,435            "unique_texts": 5765436        },437        "queries_image_statistics": null,438        "relevant_docs_statistics": {439            "num_relevant_docs": 6000,440            "min_relevant_docs_per_query": 1,441            "average_relevant_docs_per_query": 1.0,442            "max_relevant_docs_per_query": 1,443            "unique_relevant_docs": 6000444        },445        "top_ranked_statistics": null,446        "hf_subset_descriptive_stats": {447            "python": {448                "num_samples": 2000,449                "number_of_characters": 1329388,450                "documents_text_statistics": {451                    "total_text_length": 862842,452                    "min_text_length": 91,453                    "average_text_length": 862.842,454                    "max_text_length": 10914,455                    "unique_texts": 1000456                },457                "documents_image_statistics": null,458                "queries_text_statistics": {459                    "total_text_length": 466546,460                    "min_text_length": 8,461                    "average_text_length": 466.546,462                    "max_text_length": 8636,463                    "unique_texts": 982464                },465                "queries_image_statistics": null,466                "relevant_docs_statistics": {467                    "num_relevant_docs": 1000,468                    "min_relevant_docs_per_query": 1,469                    "average_relevant_docs_per_query": 1.0,470                    "max_relevant_docs_per_query": 1,471                    "unique_relevant_docs": 1000472                },473                "top_ranked_statistics": null474            },475            "javascript": {476                "num_samples": 2000,477                "number_of_characters": 1601650,478                "documents_text_statistics": {479                    "total_text_length": 1415632,480                    "min_text_length": 95,481                    "average_text_length": 1415.632,482                    "max_text_length": 334374,483                    "unique_texts": 1000484                },485                "documents_image_statistics": null,486                "queries_text_statistics": {487                    "total_text_length": 186018,488                    "min_text_length": 2,489                    "average_text_length": 186.018,490                    "max_text_length": 7657,491                    "unique_texts": 951492                },493                "queries_image_statistics": null,494                "relevant_docs_statistics": {495                    "num_relevant_docs": 1000,496                    "min_relevant_docs_per_query": 1,497                    "average_relevant_docs_per_query": 1.0,498                    "max_relevant_docs_per_query": 1,499                    "unique_relevant_docs": 1000500                },501                "top_ranked_statistics": null502            },503            "go": {504                "num_samples": 2000,505                "number_of_characters": 688942,506                "documents_text_statistics": {507                    "total_text_length": 563729,508                    "min_text_length": 69,509                    "average_text_length": 563.729,510                    "max_text_length": 15904,511                    "unique_texts": 1000512                },513                "documents_image_statistics": null,514                "queries_text_statistics": {515                    "total_text_length": 125213,516                    "min_text_length": 14,517                    "average_text_length": 125.213,518                    "max_text_length": 1501,519                    "unique_texts": 988520                },521                "queries_image_statistics": null,522                "relevant_docs_statistics": {523                    "num_relevant_docs": 1000,524                    "min_relevant_docs_per_query": 1,525                    "average_relevant_docs_per_query": 1.0,526                    "max_relevant_docs_per_query": 1,527                    "unique_relevant_docs": 1000528                },529                "top_ranked_statistics": null530            },531            "ruby": {532                "num_samples": 2000,533                "number_of_characters": 891452,534                "documents_text_statistics": {535                    "total_text_length": 577634,536                    "min_text_length": 79,537                    "average_text_length": 577.634,538                    "max_text_length": 8171,539                    "unique_texts": 1000540                },541                "documents_image_statistics": null,542                "queries_text_statistics": {543                    "total_text_length": 313818,544                    "min_text_length": 5,545                    "average_text_length": 313.818,546                    "max_text_length": 17533,547                    "unique_texts": 978548                },549                "queries_image_statistics": null,550                "relevant_docs_statistics": {551                    "num_relevant_docs": 1000,552                    "min_relevant_docs_per_query": 1,553                    "average_relevant_docs_per_query": 1.0,554                    "max_relevant_docs_per_query": 1,555                    "unique_relevant_docs": 1000556                },557                "top_ranked_statistics": null558            },559            "java": {560                "num_samples": 2000,561                "number_of_characters": 1110647,562                "documents_text_statistics": {563                    "total_text_length": 420287,564                    "min_text_length": 106,565                    "average_text_length": 420.287,566                    "max_text_length": 9142,567                    "unique_texts": 1000568                },569                "documents_image_statistics": null,570                "queries_text_statistics": {571                    "total_text_length": 690360,572                    "min_text_length": 2,573                    "average_text_length": 690.36,574                    "max_text_length": 6473,575                    "unique_texts": 956576                },577                "queries_image_statistics": null,578                "relevant_docs_statistics": {579                    "num_relevant_docs": 1000,580                    "min_relevant_docs_per_query": 1,581                    "average_relevant_docs_per_query": 1.0,582                    "max_relevant_docs_per_query": 1,583                    "unique_relevant_docs": 1000584                },585                "top_ranked_statistics": null586            },587            "php": {588                "num_samples": 2000,589                "number_of_characters": 874248,590                "documents_text_statistics": {591                    "total_text_length": 712129,592                    "min_text_length": 108,593                    "average_text_length": 712.129,594                    "max_text_length": 15584,595                    "unique_texts": 1000596                },597                "documents_image_statistics": null,598                "queries_text_statistics": {599                    "total_text_length": 162119,600                    "min_text_length": 5,601                    "average_text_length": 162.119,602                    "max_text_length": 1240,603                    "unique_texts": 911604                },605                "queries_image_statistics": null,606                "relevant_docs_statistics": {607                    "num_relevant_docs": 1000,608                    "min_relevant_docs_per_query": 1,609                    "average_relevant_docs_per_query": 1.0,610                    "max_relevant_docs_per_query": 1,611                    "unique_relevant_docs": 1000612                },613                "top_ranked_statistics": null614            }615        }616    }617}618```619 620</details>621 622---623*This dataset card was automatically generated using [MTEB](https://github.com/embeddings-benchmark/mteb)*