google/code_x_glue_cc_code_refinement
Dataset Card for "code_x_glue_cc_code_refinement" Dataset Summary CodeXGLUE code-refinement dataset, available at https://github.com/microsoft/CodeXGLUE/tree/main/Code-Code/code-refinement We use the dataset released by this paper(https://arxiv.org/pdf/1812.08693.pdf). The source side is a Java function with bugs and the target side is the refined one. All the function and variable names are normalized. Their dataset contains two subsets ( i.e.small and medium)… See the full description on the dataset page: https://huggingface.co/datasets/google/code_x_glue_cc_code_refinement.
81k
1from typing import List2 3import datasets4 5 6# Citation, taken from https://github.com/microsoft/CodeXGLUE7_DEFAULT_CITATION = """@article{CodeXGLUE,8 title={CodeXGLUE: A Benchmark Dataset and Open Challenge for Code Intelligence},9 year={2020},}"""10 11 12class Child:13 _DESCRIPTION = None14 _FEATURES = None15 _CITATION = None16 SPLITS = {"train": datasets.Split.TRAIN}17 _SUPERVISED_KEYS = None18 19 def __init__(self, info):20 self.info = info21 22 def homepage(self):23 return self.info["project_url"]24 25 def _info(self):26 # This is the description that will appear on the datasets page.27 return datasets.DatasetInfo(28 description=self.info["description"] + "\n\n" + self._DESCRIPTION,29 features=datasets.Features(self._FEATURES),30 homepage=self.homepage(),31 citation=self._CITATION or _DEFAULT_CITATION,32 supervised_keys=self._SUPERVISED_KEYS,33 )34 35 def _split_generators(self, dl_manager: datasets.DownloadManager) -> List[datasets.SplitGenerator]:36 SPLITS = self.SPLITS37 _URL = self.info["raw_url"]38 urls_to_download = {}39 for split in SPLITS:40 if split not in urls_to_download:41 urls_to_download[split] = {}42 43 for key, url in self.generate_urls(split):44 if not url.startswith("http"):45 url = _URL + "/" + url46 urls_to_download[split][key] = url47 48 downloaded_files = {}49 for k, v in urls_to_download.items():50 downloaded_files[k] = dl_manager.download(v)51 52 return [53 datasets.SplitGenerator(54 name=SPLITS[k],55 gen_kwargs={"split_name": k, "file_paths": downloaded_files[k]},56 )57 for k in SPLITS58 ]59 60 def check_empty(self, entries):61 all_empty = all([v == "" for v in entries.values()])62 all_non_empty = all([v != "" for v in entries.values()])63 64 if not all_non_empty and not all_empty:65 raise RuntimeError("Parallel data files should have the same number of lines.")66 67 return all_empty68 69 70class TrainValidTestChild(Child):71 SPLITS = {72 "train": datasets.Split.TRAIN,73 "valid": datasets.Split.VALIDATION,74 "test": datasets.Split.TEST,75 }76