ymoslem/UN-Arabic-English-Filtered
Dataset Details MultiUN + UNPC datasets, with rule-based and semantic filtering (train > 0.45 - test/dev > 0.9) as well as (>= 0.1) fasttext language detection. Dataset Structure DatasetDict({ train: Dataset({ features: ['text_en', 'text_ar'], num_rows: 19279407 }) test: Dataset({ features: ['text_en', 'text_ar'], num_rows: 8752 }) dev: Dataset({ features: ['text_en', 'text_ar'], num_rows: 8752… See the full description on the dataset page: https://huggingface.co/datasets/ymoslem/UN-Arabic-English-Filtered.
Dataset Details
MultiUN + UNPC datasets, with rule-based and semantic filtering (train > 0.45 - test/dev > 0.9) as well as (>= 0.1) fasttext language detection.
Dataset Structure
DatasetDict({
train: Dataset({
features: ['text_en', 'text_ar'],
num_rows: 19279407
})
test: Dataset({
features: ['text_en', 'text_ar'],
num_rows: 8752
})
dev: Dataset({
features: ['text_en', 'text_ar'],
num_rows: 8752
})
})Citations
@inproceedings{eisele-chen-2010-multiun,
title = "{M}ulti{UN}: A Multilingual Corpus from United Nation Documents",
author = "Eisele, Andreas and
Chen, Yu",
editor = "Calzolari, Nicoletta and
Choukri, Khalid and
Maegaard, Bente and
Mariani, Joseph and
Odijk, Jan and
Piperidis, Stelios and
Rosner, Mike and
Tapias, Daniel",
booktitle = "Proceedings of the Seventh International Conference on Language Resources and Evaluation ({LREC}'10)",
month = may,
year = "2010",
address = "Valletta, Malta",
publisher = "European Language Resources Association (ELRA)",
url = "http://www.lrec-conf.org/proceedings/lrec2010/pdf/686_Paper.pdf",
}@inproceedings{ziemski-etal-2016-united,
title = "The {U}nited {N}ations Parallel Corpus v1.0",
author = "Ziemski, Micha{\l} and
Junczys-Dowmunt, Marcin and
Pouliquen, Bruno",
editor = "Calzolari, Nicoletta and
Choukri, Khalid and
Declerck, Thierry and
Goggi, Sara and
Grobelnik, Marko and
Maegaard, Bente and
Mariani, Joseph and
Mazo, Helene and
Moreno, Asuncion and
Odijk, Jan and
Piperidis, Stelios",
booktitle = "Proceedings of the Tenth International Conference on Language Resources and Evaluation ({LREC}'16)",
month = may,
year = "2016",
address = "Portoro{\v{z}}, Slovenia",
publisher = "European Language Resources Association (ELRA)",
url = "https://aclanthology.org/L16-1561",
pages = "3530--3534",
}@INPROCEEDINGS{Tiedemann2012-OPUS,
title = "{Parallel Data, Tools and Interfaces in {OPUS}}",
booktitle = "{Proceedings of the Eighth International Conference on Language
Resources and Evaluation ({LREC}'12)}",
author = "Tiedemann, J{\"o}rg",
publisher = "European Language Resources Association (ELRA)",
pages = "2214--2218",
month = may,
year = 2012,
url = "http://www.lrec-conf.org/proceedings/lrec2012/pdf/463_Paper.pdf",
address = "Istanbul, Turkey"
}