RASMUS/Finnish-ASR-Canary-v2

1# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import argparse16import json17import logging18import os19 20import librosa21import numpy as np22from joblib import Parallel, delayed23from scipy.io import wavfile24from tqdm import tqdm25from nemo.collections.asr.parts.utils.manifest_utils import read_manifest26 27parser = argparse.ArgumentParser(description='Create synthetic code-switching data audio data from monolingual data')28parser.add_argument("--manifest_path", default=None, type=str, help='Path to CS indermediate manifest', required=True)29parser.add_argument(30    "--audio_save_folder_path",31    default=None,32    type=str,33    help='Path to directory where created synthetic set would be saved',34    required=True,35)36parser.add_argument(37    "--manifest_save_path", default=None, type=str, help='Path to save the created manifest', required=True38)39parser.add_argument(40    "--audio_normalized_amplitude", default=15000, type=int, help='Normalized amplitdue of audio samples'41)42parser.add_argument(43    "--cs_data_sampling_rate",44    default=16000,45    type=int,46    help='Desired sampling rate for the audios in the generated dataset',47)48parser.add_argument(49    "--sample_beginning_pause_msec",50    default=20,51    type=int,52    help='Pause to be added at the beginning of the sample (msec)',53)54parser.add_argument(55    "--sample_joining_pause_msec",56    default=100,57    type=int,58    help='Pause to be added between different phrases of the sample (msec)',59)60parser.add_argument(61    "--sample_end_pause_msec", default=20, type=int, help='Pause to be added at the end of the sample (msec)'62)63parser.add_argument(64    "--is_lid_manifest",65    default=True,66    type=bool,67    help='If true, generate manifest in the multi-sample lid format, else the standard manifest format',68)69parser.add_argument("--workers", default=1, type=int, help='Number of worker processes')70 71args = parser.parse_args()72 73 74def split_list(input_list: list, num_splits: int):75    """76    Args:77        input_list: the input list to split78        num_splits: number of splits required79 80    Returns:81        iterator of split lists82 83    """84    k, m = divmod(len(input_list), num_splits)85    return (input_list[i * k + min(i, m) : (i + 1) * k + min(i + 1, m)] for i in range(num_splits))86 87 88def combine_manifests(manifest_save_path: str, num_split: int):89    """90    Args:91        manifest_save_path: absolute path to save the combined manifest92        num_splits: number of splits of manifest93 94    Returns:95        num_samples_combined: the total number of samples in the generated dataset96    """97    num_samples_combined = 098    base_directory = os.path.dirname(manifest_save_path)99 100    with open(manifest_save_path, 'w') as outfile:101        for i in range(num_split):102            split_manifest_path = base_directory + '/temp_' + str(i) + '.json'103            data_split = read_manifest(split_manifest_path)104 105            for elem in data_split:106                s = json.dumps(elem)107                outfile.write(s + '\n')108                num_samples_combined += 1109 110            # removing the intermediate file111            os.remove(split_manifest_path)112 113    return num_samples_combined114 115 116def create_cs_data(117    intermediate_cs_manifest_list: list,118    audio_save_folder: str,119    manfest_save_path: str,120    audio_amplitude_normalization: int,121    pause_beg_msec: int,122    pause_join_msec: int,123    pause_end_msec: int,124    cs_data_sampling_rate: int,125    is_lid_manifest: bool,126):127 128    """129    Args:130        intermediate_cs_manifest_list: the intermediate cs manifest obtained from code_switching_manifest_creation.py as a list131        audio_save_folder: Absolute path to save the generated audio samples132        manfest_save_path: Absolute path to save the corresponding manifest133        audio_amplitude_normalization: The amplitude to scale to after normalization134        pause_beg_msec: Pause to be added at the beginning of the sample (msec)135        pause_join_msec: Pause to be added between different phrases of the sample (msec)136        pause_end_msec: Pause to be added at the end of the sample (msec)137        cs_data_sampling_rate: Desired sampling rate of the generated samples138        is_lid_manifest: If true, generate manifest in the multi-sample lid format, else the standard manifest format139 140    Returns:141 142    """143 144    fs = cs_data_sampling_rate145    incorrect_sample_flag = 0146 147    with open(manfest_save_path, 'w') as outfile:148        for data in tqdm(intermediate_cs_manifest_list):149 150            combined_audio = []151 152            staring_pause = np.zeros(int(pause_beg_msec * fs / 1000))153            combined_audio += list(staring_pause)154 155            text_entry_list = []156            for index in range(len(data['lang_ids'])):157 158                phrase_entry = {}159                # dictionary to store the phrase information which will be added to the complete sentence160 161                data_sample, fs_sample = librosa.load(data['paths'][index], sr=fs)162                # Alternative-  fs_sample, data_sample = wavfile.read(data['paths'][index])163 164                if fs_sample != fs:165                    logging.error('Sampling rate error inside create_cs_data function')166                    exit167 168                # Remove leading and trailing zeros169                data_sample = np.trim_zeros(data_sample)170 171                # take care of empty arrays: rare172                if data_sample.size == 0:173                    incorrect_sample_flag = 1174                    continue175 176                # normalizing data177                data_sample_norm = (178                    data_sample179                    / np.maximum(np.abs(data_sample.max()), np.abs(data_sample.min()))180                    * audio_amplitude_normalization181                )182 183                combined_audio += list(data_sample_norm)184 185                phrase_entry['str'] = data['texts'][index]186                phrase_entry['lang'] = data['lang_ids'][index]187 188                text_entry_list.append(phrase_entry)189 190                # adding small pause between semgments191                if index != (len(data['lang_ids']) - 1):192                    pause = np.zeros(int(pause_join_msec * fs / 1000))193                    combined_audio += list(pause)194 195            if incorrect_sample_flag == 1:196                incorrect_sample_flag = 0197                continue198 199            ending_pause = np.zeros(int(pause_end_msec * fs / 1000))200            combined_audio += list(ending_pause)201 202            sample_id = data['uid']203            audio_file_path = audio_save_folder + '/' + str(sample_id) + ".wav"204 205            # saving audio206            wavfile.write(audio_file_path, fs, np.array(combined_audio).astype(np.int16))207            # Alternative-  librosa.output.write_wav(audio_file_path, combined_audio, fs)208 209            metadata_json = {}210            metadata_json['audio_filepath'] = audio_file_path211            metadata_json['duration'] = float(len(combined_audio) / fs)212            if is_lid_manifest:213                metadata_json['text'] = text_entry_list214            else:215                metadata_json['text'] = ' '.join(data['texts'])216 217            metadata_json['language_ids'] = data['lang_ids']218            metadata_json['original_texts'] = data['texts']219            metadata_json['original_paths'] = data['paths']220            metadata_json['original_durations'] = data['durations']221 222            s = json.dumps(metadata_json)223            outfile.write(s + '\n')224 225 226def main():227 228    cs_intermediate_manifest_path = args.manifest_path229    audio_save_folder = args.audio_save_folder_path230    manifest_save_path = args.manifest_save_path231    audio_amplitude_normalization = args.audio_normalized_amplitude232    pause_beg_msec = args.sample_beginning_pause_msec233    pause_join_msec = args.sample_joining_pause_msec234    pause_end_msec = args.sample_end_pause_msec235    cs_data_sampling_rate = args.cs_data_sampling_rate236    is_lid_manifest = args.is_lid_manifest237    num_process = args.workers238 239    # Sanity Checks240    if (cs_intermediate_manifest_path is None) or (not os.path.exists(cs_intermediate_manifest_path)):241        logging.error('Please provide correct CS manifest (obtained from code_switching_manifest_creation.py)')242        exit243 244    if (audio_save_folder is None) or (not os.path.exists(audio_save_folder)):245        logging.error('audio_save_folder_path is incorrect or does not exist')246        exit247 248    if manifest_save_path is None:249        logging.error('Please provide valid manifest_save_path')250        exit251 252    # Reading data253    logging.info('Reading manifests')254    intermediate_cs_manifest = read_manifest(cs_intermediate_manifest_path)255 256    # Spliting the data257    data_split = split_list(intermediate_cs_manifest, num_process)258 259    # Creating Audio data260    logging.info('Creating synthetic audio data')261    base_directory = os.path.dirname(manifest_save_path)262 263    Parallel(n_jobs=num_process)(264        delayed(create_cs_data)(265            split_manifest,266            audio_save_folder,267            base_directory + '/temp_' + str(idx) + '.json',268            audio_amplitude_normalization,269            pause_beg_msec,270            pause_join_msec,271            pause_end_msec,272            cs_data_sampling_rate,273            is_lid_manifest,274        )275        for idx, split_manifest in enumerate(data_split)276    )277 278    # Combining manifests279    num_samples_combined = combine_manifests(manifest_save_path, num_process)280 281    print("Synthetic CS audio data saved at :", audio_save_folder)282    print("Synthetic CS manifest saved at :", manifest_save_path)283    print("Total number of samples in the generated dataset :", str(num_samples_combined))284 285    logging.info('Done!')286 287 288if __name__ == "__main__":289    main()290