SetFit/amazon_massive_scenario_ta-IN

1from datasets import load_dataset2from huggingface_hub import create_repo, Repository, upload_file3import os4import typer5 6def main(language_label):7	raw_data = load_dataset("AmazonScience/massive", language_label)8	raw_data = raw_data.rename_column("utt", "text")9	raw_data = raw_data.rename_column("scenario", "label")10	raw_data = raw_data.remove_columns(["locale", "partition", "intent", "annot_utt", 11    									"slot_method", "worker_id", "judgments"])12 13	#to get labels14	labels = raw_data["train"].features["label"]15	16	#for uploading to hub17	repo_name = "amazon_massive_scenario_" + language_label18	create_repo(repo_name, organization="SetFit", repo_type="dataset")19 20	for split, dataset in raw_data.items():21		dataset = dataset.map(lambda x: {"label_text": labels.int2str(x["label"])}, num_proc=4)22		dataset.to_json(f"{split}.jsonl")23		upload_file(f"{split}.jsonl", path_in_repo=f"{split}.jsonl", repo_id="SetFit/" + repo_name, repo_type="dataset")24		os.system(f"rm {split}.jsonl")25 26	upload_file("create_dataset.py", path_in_repo="create_dataset.py", repo_id="SetFit/" + repo_name, repo_type="dataset")27	28	29 30 31if __name__ == "__main__":32	typer.run(main)33 34 35