Datasetpublic

cw1521/ember2018-malware

EMBER 2018 Malware Analysis Dataset This dataset contains 1 million records of metadata and vectorized features for malware and benign software. Visit https://github.com/elastic/ember for more information on the dataset. Usage dataset = load_dataset("cw1521/ember2018-malware", field="data") input - vectorized features as a string label - (0 for benign and 1 for malware)

sourceHugging Faceupdated 2y agoView on Hugging Face

10likes1.4kdownloads

rootbuild_dataset.py

build_dataset.py136 linesDownload Raw Back to root

1 2import json3import os4 5import datasets6 7 8 9_CITATION = """\10@InProceedings{huggingface:dataset,11title = {Ember2018},12author=Christian Williams13},14year={2023}15}16"""17 18_DESCRIPTION = """\19This dataset is from the EMBER 2018 Malware Analysis dataset20"""21_HOMEPAGE = "https://github.com/elastic/ember"22_LICENSE = ""23_URLS = {24    "text_classification": "https://huggingface.co/datasets/cw1521/ember2018-malware/blob/main/data/"25}26 27 28class EMBERConfig(datasets.GeneratorBasedBuilder):29    VERSION = datasets.Version("1.1.0")30    BUILDER_CONFIGS = [31        datasets.BuilderConfig(32            name="text_classification", 33            version=VERSION, description="This part of my dataset covers text classification"34        )35    ]36 37    DEFAULT_CONFIG_NAME = "text_classification" 38 39    def _info(self):40        if self.config.name == "text_classification": 41            features = datasets.Features(42                {43                    "input": datasets.Value("string"),44                    "label": datasets.Value("string"),45                    "x": datasets.features.Sequence(46                            datasets.Value("float32")47                    ),48                    "y": datasets.Value("string"),49                    "appeared": datasets.Value("string"),50                    "avclass": datasets.Value("string"),51                    "subset": datasets.Value("string"),52                    "sha256": datasets.Value("string")53                }54            )55        else: 56            features = datasets.Features(57                {58                    "input": datasets.Value("string"),59                    "label": datasets.Value("string"),60                    "x": datasets.features.Sequence(61                            datasets.Value("float32")62                    ),63                    "y": datasets.Value("string"),64                    "appeared": datasets.Value("string"),65                    "avclass": datasets.Value("string"),66                    "subset": datasets.Value("string"),67                    "sha256": datasets.Value("string")68                }69            )70        return datasets.DatasetInfo(71            description=_DESCRIPTION,72            features=features, 73            homepage=_HOMEPAGE,74            license=_LICENSE,75            citation=_CITATION,76        )77        78    def _split_generators(self, dl_manager):79        urls = _URLS[self.config.name]80        data_dir = dl_manager.download_and_extract(urls)81        return [82            datasets.SplitGenerator(83                name=datasets.Split.TRAIN,84                gen_kwargs={85                    "filepaths": os.path.join(data_dir, "ember2018_train_*.jsonl"),86                    "split": "train",87                },88            ),89            # datasets.SplitGenerator(90            #     name=datasets.Split.VALIDATION,91            #     gen_kwargs={92            #         "filepaths": os.path.join(data_dir, "*_valid_*.jsonl"),93            #         "split": "valid",94            #     },95            # ),96            datasets.SplitGenerator(97                name=datasets.Split.TEST,98                gen_kwargs={99                    "filepaths": os.path.join(data_dir, "ember2018_test_*.jsonl"),100                    "split": "test"101                },102            )103        ]104 105 106    def _generate_examples(self, filepaths, split):107        key = 0108        for id, filepath in enumerate(filepaths[split]): 109            key += 1110            with open(filepath[id], encoding="utf-8") as f:111                data_list = json.load(f)112                for data in data_list:113                    if self.config.name == "text_classification":114                        data.remove115                        yield key, {116                            "input": data["input"],117                            "label": data["label"],118                            # "x": data["x"],119                            # "y": data["y"],120                            # "appeared": data["appeared"],121                            # "avclass": data["avclass"],122                            # "subset": data["subset"],123                            # "sha256": data["sha256"]124                        }125                    else:126                        yield key, {127                            "input": data["input"],128                            "label": data["label"],129                            "x": data["x"],130                            "y": data["y"],131                            "appeared": data["appeared"],132                            "avclass": data["avclass"],133                            "subset": data["subset"],134                            "sha256": data["sha256"]135                        }136