Datasetpublic

mteb/multi-hatecheck

MultiHateClassification An MTEB dataset Massive Text Embedding Benchmark Hate speech detection dataset with binary (hateful vs non-hateful) labels. Includes 25+ distinct types of hate and challenging non-hate, and 11 languages. Task category t2c Domains Constructed, Written Reference https://aclanthology.org/2022.woah-1.15/ How to evaluate on this task You can evaluate an embedding model on this dataset… See the full description on the dataset page: https://huggingface.co/datasets/mteb/multi-hatecheck.

sourceHugging Facecc-by-4.0updated 7mo agoView on Hugging Face

0likes701downloads

rootscriptsmake_dataset.py

make_dataset.py75 linesDownload Raw Back to scripts

1from __future__ import annotations2 3from pathlib import Path4import numpy as np5 6import datasets7 8 9_HF_AFFIX = {10    "ara": "arabic",11    "cmn": "mandarin",12    "eng": "",13    "deu": "german",14    "fra": "french",15    "hin": "hindi",16    "ita": "italian",17    "nld": "dutch",18    "pol": "polish",19    "por": "portuguese",20    "spa": "spanish",21}22 23_HF_AFFIX_REV = {v:k for k,v in _HF_AFFIX.items()}24 25_REVISION_DICT = {26    "ara": "65eb7455a05cb77b3ae0c69d444569a8eee54628",27    "cmn": "617d3e9fccd186277297cc305f6588af7384b008",28    "eng": "9d2ac89df04254e5c427bcc8d61b6d6c83a1f59b",29    "deu": "5229a5cc475f36c08d03ca52f0ccb005705e60d2",30    "fra": "5d3085f2129139abc10d2b58becd4d4f2978e5d5",31    "hin": "e9e68e1a4db04726b9278192377049d0f9693012",32    "ita": "21e3d5c827cb60619a89988b24979850a7af85a5",33    "nld": "d622427417d37a8d74e110e6289bc29af4ba4056",34    "pol": "28d7098e2e5a211c4810d0a4d8deccc5889e55b6",35    "por": "323bdf67e0fbd3d7f8086fad0971b5bd5a62524b",36    "spa": "a7ea759535bb9fad6361cca151cf94a46e88edf3",37}38 39def _transform(dataset):40    target_cols = ["test_case", "label_gold"]41    new_cols = ['text', 'is_hateful']42    rename_dict = dict(zip(target_cols, ["text", "is_hateful"]))43    dataset = dataset.rename_columns(rename_dict)44    keep_cols = new_cols + ["functionality"]45    remove_cols = [col for col in dataset["test"].column_names if col not in keep_cols]46    dataset = dataset.remove_columns(remove_cols)47    return dataset48 49 50def make_dataset():51    """52    Load dataset from HuggingFace hub53    """54    ds = {}55    for lang in _HF_AFFIX.values():56        lcode = _HF_AFFIX_REV[lang]57        path = f'Paul/hatecheck-{lang}'.rstrip('-')58        dataset = datasets.load_dataset(59            path=path, revision=_REVISION_DICT[lcode]60        )61        dataset = _transform(dataset)62        out_path = Path('..') / lcode / 'test.jsonl'63        dataset['test'].to_json(out_path)64        ds[lcode] = dataset65    return ds66    67 68if __name__ == '__main__':69    dataset = make_dataset()70    AVG_CHAR = 071    for lang in _HF_AFFIX:72        AVG_CHAR += np.mean([len(x['text']) for x in dataset[lang]['test']])73    print(f'avg char: {AVG_CHAR / len(_HF_AFFIX)}')74 75