23ws-LLMcoder/LLMcoder-GitHub-Python-Mix-Direct
Dataset Card for LLMcoder-GitHub-Python-Mix-Direct Python target autocomplete suggestions in the format of conversations for OpenAI's fine-tuning. Dataset Details Dataset Description Curated by: [More Information Needed] Funded by [optional]: [More Information Needed] Shared by [optional]: [More Information Needed] Language(s) (NLP): [More Information Needed] License: [More Information Needed] Dataset Sources [optional] The data… See the full description on the dataset page: https://huggingface.co/datasets/23ws-LLMcoder/LLMcoder-GitHub-Python-Mix-Direct.
0216
1import json2import os3import shutil4 5import numpy as np6import pandas as pd7import urllib38from tqdm import tqdm9 10from ..utils import get_data_dir, get_labels11from .preprocessing import get_granularity, get_type, stringify_context12 13 14def download_data(target_dir: str = None) -> None:15 """16 Download the Ultra-Fine Entity Typing (ACL 2018) data from the web.17 For more information, see https://www.cs.utexas.edu/~eunsol/html_pages/open_entity.html18 19 Parameters20 ----------21 target_dir : str, optional22 The directory to download the data to. The default is 'data'.23 """24 DATA_URL = 'http://nlp.cs.washington.edu/entity_type/data/ultrafine_acl18.tar.gz'25 26 if target_dir is None:27 target_dir = get_data_dir()28 29 print(f'Downloading the data from {DATA_URL} to {target_dir}...')30 31 # Create the data directory if it does not exist32 os.makedirs(target_dir, exist_ok=True)33 34 # Download the data35 http = urllib3.PoolManager()36 with http.request('GET', DATA_URL, preload_content=False) as r, open('dat