Blablablab/audio-classification

1"""2Multilingual Configuration Setup Module3 4This module provides functionality for setting up multilingual annotation tasks.5It processes a base configuration and creates language-specific configurations6for each supported language, including:7 8- Language-specific survey flow files9- Translated configuration files10- Localized output directories11- Multilingual guideline integration12 13The module supports dynamic text replacement using key-based translation mappings14and creates the necessary directory structure for multilingual annotation projects.15"""16 17from argparse import ArgumentParser18import yaml19import os20import pandas as pd21import json22from collections import defaultdict23 24 25def arguments():26    """27    Creates and returns the argument parser for multilingual configuration setup.28 29    Returns:30        ArgumentParser: Configured argument parser with multilingual_config_file argument31    """32    parser = ArgumentParser()33    parser.set_defaults(show_path=False, show_similarity=False)34 35    parser.add_argument("multilingual_config_file")36 37    return parser.parse_args()38 39 40def main():41    """42    Main function for setting up multilingual annotation configurations.43 44    This function processes a multilingual configuration file and creates45    language-specific configurations for each supported language. It handles:46 47    1. Directory structure creation48    2. Translation mapping from guideline files49    3. Configuration file generation for each language50    4. Survey flow file localization51    5. Output directory setup52 53    Side Effects:54        - Creates directories for surveyflow, annotation_output, data_files, configs, htmls55        - Generates language-specific configuration files56        - Creates localized survey flow files57        - Sets up language-specific output directories58    """59    args = arguments()60 61    # Load multilingual annotation configuration62    with open(args.multilingual_config_file, "rt") as f:63        multilingual_config = yaml.safe_load(f)64 65    # Create basic folder structure for the multilingual project66    for folder in ["surveyflow", "annotation_output", "data_files", "configs", "htmls"]:67        cur_path = multilingual_config["base_dir"] + folder68        if not os.path.exists(cur_path):69            os.makedirs(cur_path)70            print("Created directory: %s" % (cur_path))71 72    # Build translation mapping dictionary73    # This maps translation keys to their language-specific text values74    key2text = defaultdict(dict)75 76    # Load multilingual annotation guidelines if specified and available77    if "multilingual_guideline_file" in multilingual_config and os.path.exists(78        multilingual_config["multilingual_guideline_file"]79    ):80        # Load multilingual annotation guideline from CSV81        multilingual_guideline_df = pd.read_csv(multilingual_config["multilingual_guideline_file"])82        for i, row in multilingual_guideline_df.iterrows():83            # Only process rows with proper key format (enclosed in brackets)84            if type(row["key"]) != str or row["key"][0] != "[" or row["key"][-1] != "]":85                continue86            # Create translation mapping for each supported language87            for lang in multilingual_config["languages"]:88                key2text[row["key"]][lang] = (89                    row[lang]90                    if type(row[lang]) == str91                    else row[multilingual_config["base_language"]]92                )93 94    # Generate configuration for each supported language95    for lang in multilingual_config["languages"]:96        # Load and process the base configuration file97        with open(multilingual_config["base_config_file"], "rt") as f:98            page = f.read()99            # Replace translation keys with language-specific text100            for key in key2text:101                page = page.replace(key, key2text[key][lang])102 103            # Update surveyflow output path for this language104            surveyflow_output_path = multilingual_config["surveyflow_output_path"].replace(105                "[LANGUAGE]", lang106            )107            page = page.replace(108                multilingual_config["surveyflow_path"], surveyflow_output_path + lang + "-"109            )110            page = page.replace("[LANGUAGE]", lang)111 112            # Parse the processed configuration113            config = yaml.safe_load(page)114 115        # Set up language-specific output directory116        config["output_annotation_dir"] = multilingual_config["output_annotation_dir"].replace(117            "[LANGUAGE]", lang118        )119        if not os.path.exists(config["output_annotation_dir"]):120            os.makedirs(config["output_annotation_dir"])121 122        """123        # setup the site_dir path for each language124        config['path_under_site_dir'] = multilingual_config['path_under_site_dir'].replace("[LANGUAGE]", lang)125        config["site_dir"] += config['path_under_site_dir']126        if not os.path.exists(config["site_dir"]):127            os.makedirs(config["site_dir"])128        """129 130        # Update task name and data files for this language131        config["annotation_task_name"] = multilingual_config["annotation_task_name"].replace(132            "[LANGUAGE]", lang133        )134        config["data_files"] = [135            it.replace("[LANGUAGE]", lang) for it in multilingual_config["data_files"]136        ]137 138        # Save the language-specific configuration file139        with open(multilingual_config["base_dir"] + "configs/%s.yaml" % lang, "wt") as f:140            json.dump(config, f, indent=4)141 142        # Create the directory for surveyflow output path143        if not os.path.exists(surveyflow_output_path):144            os.makedirs(surveyflow_output_path)145 146        # Process and localize surveyflow files147        surveyflow_files = os.listdir(multilingual_config["surveyflow_path"])148        for file in surveyflow_files:149            # Skip directories, only process files150            if os.path.isdir(multilingual_config["surveyflow_path"] + file):151                continue152            # Read the surveyflow file153            with open(multilingual_config["surveyflow_path"] + file, "r", encoding="utf-8") as f:154                page = f.read()155            # Replace translation keys with language-specific text156            for key in key2text:157                page = page.replace(key, key2text[key][lang])158            page = page.replace("[LANGUAGE]", lang)159            # Write the localized surveyflow file160            with open(surveyflow_output_path + lang + "-" + file, "wt", encoding="utf-8") as f:161                f.write(page)162 163        # for key in ["surveyflow_output_path", ]164 165 166if __name__ == "__main__":167    main()168