Ahmed007/hamsa-tiny-v0.4

1from datasets import load_dataset, DatasetDict2from transformers import WhisperFeatureExtractor3from transformers import WhisperTokenizer4from transformers import WhisperProcessor5from datasets import Audio6from transformers.models.whisper.english_normalizer import BasicTextNormalizer7from huggingface_hub import login8 9import argparse10 11my_parser = argparse.ArgumentParser()12 13my_parser.add_argument(14    "--model_name",15    "-model_name",16    type=str,17    action="store",18    default="openai/whisper-tiny",19)20my_parser.add_argument("--hf_token", "-hf_token", type=str, action="store")21my_parser.add_argument(22    "--dataset_name", "-dataset_name", type=str, action="store", default="google/fleurs"23)24my_parser.add_argument("--split", "-split", type=str, action="store", default="test")25my_parser.add_argument("--subset", "-subset", type=str, action="store")26 27args = my_parser.parse_args()28 29dataset_name = args.dataset_name30model_name = args.model_name31subset = args.subset32hf_token = args.hf_token33login(hf_token)34text_column = "sentence"35if dataset_name == "google/fleurs":36    text_column = "transcription"37 38do_lower_case = False39do_remove_punctuation = False40 41normalizer = BasicTextNormalizer()42processor = WhisperProcessor.from_pretrained(43    model_name, language="Arabic", task="transcribe"44)45dataset = load_dataset(dataset_name, subset, use_auth_token=True)46 47print(dataset)48 49feature_extractor = WhisperFeatureExtractor.from_pretrained(model_name)50 51tokenizer = WhisperTokenizer.from_pretrained(52    model_name, language="Arabic", task="transcribe"53)54dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))55 56 57def prepare_dataset(batch):58    # load and (possibly) resample audio data to 16kHz59    audio = batch["audio"]60 61    # compute log-Mel input features from input audio array62    batch["input_features"] = processor.feature_extractor(63        audio["array"], sampling_rate=audio["sampling_rate"]64    ).input_features[0]65    # compute input length of audio sample in seconds66    batch["input_length"] = len(audio["array"]) / audio["sampling_rate"]67 68    # optional pre-processing steps69    transcription = batch[text_column]70    if do_lower_case:71        transcription = transcription.lower()72    if do_remove_punctuation:73        transcription = normalizer(transcription).strip()74 75    # encode target text to label ids76    batch["labels"] = processor.tokenizer(transcription).input_ids77    return batch78 79 80dataset = dataset.map(prepare_dataset, remove_columns=dataset.column_names["train"])81 82login(hf_token)83print(84    f"pushing to arbml/{dataset_name.split('/')[-1]}_preprocessed_{model_name.split('/')[-1]}"85)86dataset.push_to_hub(87    f"arbml/{dataset_name.split('/')[-1]}_preprocessed_{model_name.split('/')[-1]}",88    private=True,89)90