chendl/compositional_test

1# coding=utf-82# Copyright 2021 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import argparse17import collections18import os19import re20import tempfile21 22import pandas as pd23from datasets import Dataset24from huggingface_hub import Repository25 26from transformers.utils import direct_transformers_import27 28 29# All paths are set with the intent you should run this script from the root of the repo with the command30# python utils/update_metadata.py31TRANSFORMERS_PATH = "src/transformers"32 33 34# This is to make sure the transformers module imported is the one in the repo.35transformers_module = direct_transformers_import(TRANSFORMERS_PATH)36 37 38# Regexes that match TF/Flax/PT model names.39_re_tf_models = re.compile(r"TF(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")40_re_flax_models = re.compile(r"Flax(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")41# Will match any TF or Flax model too so need to be in an else branch afterthe two previous regexes.42_re_pt_models = re.compile(r"(.*)(?:Model|Encoder|Decoder|ForConditionalGeneration)")43 44 45# Fill this with tuples (pipeline_tag, model_mapping, auto_model)46PIPELINE_TAGS_AND_AUTO_MODELS = [47    ("pretraining", "MODEL_FOR_PRETRAINING_MAPPING_NAMES", "AutoModelForPreTraining"),48    ("feature-extraction", "MODEL_MAPPING_NAMES", "AutoModel"),49    ("audio-classification", "MODEL_FOR_AUDIO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForAudioClassification"),50    ("text-generation", "MODEL_FOR_CAUSAL_LM_MAPPING_NAMES", "AutoModelForCausalLM"),51    ("automatic-speech-recognition", "MODEL_FOR_CTC_MAPPING_NAMES", "AutoModelForCTC"),52    ("image-classification", "MODEL_FOR_IMAGE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForImageClassification"),53    ("image-segmentation", "MODEL_FOR_IMAGE_SEGMENTATION_MAPPING_NAMES", "AutoModelForImageSegmentation"),54    ("fill-mask", "MODEL_FOR_MASKED_LM_MAPPING_NAMES", "AutoModelForMaskedLM"),55    ("object-detection", "MODEL_FOR_OBJECT_DETECTION_MAPPING_NAMES", "AutoModelForObjectDetection"),56    (57        "zero-shot-object-detection",58        "MODEL_FOR_ZERO_SHOT_OBJECT_DETECTION_MAPPING_NAMES",59        "AutoModelForZeroShotObjectDetection",60    ),61    ("question-answering", "MODEL_FOR_QUESTION_ANSWERING_MAPPING_NAMES", "AutoModelForQuestionAnswering"),62    ("text2text-generation", "MODEL_FOR_SEQ_TO_SEQ_CAUSAL_LM_MAPPING_NAMES", "AutoModelForSeq2SeqLM"),63    ("text-classification", "MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING_NAMES", "AutoModelForSequenceClassification"),64    ("automatic-speech-recognition", "MODEL_FOR_SPEECH_SEQ_2_SEQ_MAPPING_NAMES", "AutoModelForSpeechSeq2Seq"),65    (66        "table-question-answering",67        "MODEL_FOR_TABLE_QUESTION_ANSWERING_MAPPING_NAMES",68        "AutoModelForTableQuestionAnswering",69    ),70    ("token-classification", "MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING_NAMES", "AutoModelForTokenClassification"),71    ("multiple-choice", "MODEL_FOR_MULTIPLE_CHOICE_MAPPING_NAMES", "AutoModelForMultipleChoice"),72    (73        "next-sentence-prediction",74        "MODEL_FOR_NEXT_SENTENCE_PREDICTION_MAPPING_NAMES",75        "AutoModelForNextSentencePrediction",76    ),77    (78        "audio-frame-classification",79        "MODEL_FOR_AUDIO_FRAME_CLASSIFICATION_MAPPING_NAMES",80        "AutoModelForAudioFrameClassification",81    ),82    ("audio-xvector", "MODEL_FOR_AUDIO_XVECTOR_MAPPING_NAMES", "AutoModelForAudioXVector"),83    (84        "document-question-answering",85        "MODEL_FOR_DOCUMENT_QUESTION_ANSWERING_MAPPING_NAMES",86        "AutoModelForDocumentQuestionAnswering",87    ),88    (89        "visual-question-answering",90        "MODEL_FOR_VISUAL_QUESTION_ANSWERING_MAPPING_NAMES",91        "AutoModelForVisualQuestionAnswering",92    ),93    ("image-to-text", "MODEL_FOR_FOR_VISION_2_SEQ_MAPPING_NAMES", "AutoModelForVision2Seq"),94    (95        "zero-shot-image-classification",96        "MODEL_FOR_ZERO_SHOT_IMAGE_CLASSIFICATION_MAPPING_NAMES",97        "AutoModelForZeroShotImageClassification",98    ),99    ("depth-estimation", "MODEL_FOR_DEPTH_ESTIMATION_MAPPING_NAMES", "AutoModelForDepthEstimation"),100    ("video-classification", "MODEL_FOR_VIDEO_CLASSIFICATION_MAPPING_NAMES", "AutoModelForVideoClassification"),101]102 103 104# Thanks to https://stackoverflow.com/questions/29916065/how-to-do-camelcase-split-in-python105def camel_case_split(identifier):106    "Split a camelcased `identifier` into words."107    matches = re.finditer(".+?(?:(?<=[a-z])(?=[A-Z])|(?<=[A-Z])(?=[A-Z][a-z])|$)", identifier)108    return [m.group(0) for m in matches]109 110 111def get_frameworks_table():112    """113    Generates a dataframe containing the supported auto classes for each model type, using the content of the auto114    modules.115    """116    # Dictionary model names to config.117    config_maping_names = transformers_module.models.auto.configuration_auto.CONFIG_MAPPING_NAMES118    model_prefix_to_model_type = {119        config.replace("Config", ""): model_type for model_type, config in config_maping_names.items()120    }121 122    # Dictionaries flagging if each model prefix has a backend in PT/TF/Flax.123    pt_models = collections.defaultdict(bool)124    tf_models = collections.defaultdict(bool)125    flax_models = collections.defaultdict(bool)126 127    # Let's lookup through all transformers object (once) and find if models are supported by a given backend.128    for attr_name in dir(transformers_module):129        lookup_dict = None130        if _re_tf_models.match(attr_name) is not None:131            lookup_dict = tf_models132            attr_name = _re_tf_models.match(attr_name).groups()[0]133        elif _re_flax_models.match(attr_name) is not None:134            lookup_dict = flax_models135            attr_name = _re_flax_models.match(attr_name).groups()[0]136        elif _re_pt_models.match(attr_name) is not None:137            lookup_dict = pt_models138            attr_name = _re_pt_models.match(attr_name).groups()[0]139 140        if lookup_dict is not None:141            while len(attr_name) > 0:142                if attr_name in model_prefix_to_model_type:143                    lookup_dict[model_prefix_to_model_type[attr_name]] = True144                    break145                # Try again after removing the last word in the name146                attr_name = "".join(camel_case_split(attr_name)[:-1])147 148    all_models = set(list(pt_models.keys()) + list(tf_models.keys()) + list(flax_models.keys()))149    all_models = list(all_models)150    all_models.sort()151 152    data = {"model_type": all_models}153    data["pytorch"] = [pt_models[t] for t in all_models]154    data["tensorflow"] = [tf_models[t] for t in all_models]155    data["flax"] = [flax_models[t] for t in all_models]156 157    # Now let's use the auto-mapping names to make sure158    processors = {}159    for t in all_models:160        if t in transformers_module.models.auto.processing_auto.PROCESSOR_MAPPING_NAMES:161            processors[t] = "AutoProcessor"162        elif t in transformers_module.models.auto.tokenization_auto.TOKENIZER_MAPPING_NAMES:163            processors[t] = "AutoTokenizer"164        elif t in transformers_module.models.auto.feature_extraction_auto.FEATURE_EXTRACTOR_MAPPING_NAMES:165            processors[t] = "AutoFeatureExtractor"166        else:167            # Default to AutoTokenizer if a model has nothing, for backward compatibility.168            processors[t] = "AutoTokenizer"169 170    data["processor"] = [processors[t] for t in all_models]171 172    return pd.DataFrame(data)173 174 175def update_pipeline_and_auto_class_table(table):176    """177    Update the table of model class to (pipeline_tag, auto_class) without removing old keys if they don't exist178    anymore.179    """180    auto_modules = [181        transformers_module.models.auto.modeling_auto,182        transformers_module.models.auto.modeling_tf_auto,183        transformers_module.models.auto.modeling_flax_auto,184    ]185    for pipeline_tag, model_mapping, auto_class in PIPELINE_TAGS_AND_AUTO_MODELS:186        model_mappings = [model_mapping, f"TF_{model_mapping}", f"FLAX_{model_mapping}"]187        auto_classes = [auto_class, f"TF_{auto_class}", f"Flax_{auto_class}"]188        # Loop through all three frameworks189        for module, cls, mapping in zip(auto_modules, auto_classes, model_mappings):190            # The type of pipeline may not exist in this framework191            if not hasattr(module, mapping):192                continue193            # First extract all model_names194            model_names = []195            for name in getattr(module, mapping).values():196                if isinstance(name, str):197                    model_names.append(name)198                else:199                    model_names.extend(list(name))200 201            # Add pipeline tag and auto model class for those models202            table.update({model_name: (pipeline_tag, cls) for model_name in model_names})203 204    return table205 206 207def update_metadata(token, commit_sha):208    """209    Update the metadata for the Transformers repo.210    """211    with tempfile.TemporaryDirectory() as tmp_dir:212        repo = Repository(tmp_dir, clone_from="huggingface/transformers-metadata", repo_type="dataset", token=token)213 214        frameworks_table = get_frameworks_table()215        frameworks_dataset = Dataset.from_pandas(frameworks_table)216        frameworks_dataset.to_json(os.path.join(tmp_dir, "frameworks.json"))217 218        tags_dataset = Dataset.from_json(os.path.join(tmp_dir, "pipeline_tags.json"))219        table = {220            tags_dataset[i]["model_class"]: (tags_dataset[i]["pipeline_tag"], tags_dataset[i]["auto_class"])221            for i in range(len(tags_dataset))222        }223        table = update_pipeline_and_auto_class_table(table)224 225        # Sort the model classes to avoid some nondeterministic updates to create false update commits.226        model_classes = sorted(table.keys())227        tags_table = pd.DataFrame(228            {229                "model_class": model_classes,230                "pipeline_tag": [table[m][0] for m in model_classes],231                "auto_class": [table[m][1] for m in model_classes],232            }233        )234        tags_dataset = Dataset.from_pandas(tags_table)235        tags_dataset.to_json(os.path.join(tmp_dir, "pipeline_tags.json"))236 237        if repo.is_repo_clean():238            print("Nothing to commit!")239        else:240            if commit_sha is not None:241                commit_message = (242                    f"Update with commit {commit_sha}\n\nSee: "243                    f"https://github.com/huggingface/transformers/commit/{commit_sha}"244                )245            else:246                commit_message = "Update"247            repo.push_to_hub(commit_message)248 249 250def check_pipeline_tags():251    in_table = {tag: cls for tag, _, cls in PIPELINE_TAGS_AND_AUTO_MODELS}252    pipeline_tasks = transformers_module.pipelines.SUPPORTED_TASKS253    missing = []254    for key in pipeline_tasks:255        if key not in in_table:256            model = pipeline_tasks[key]["pt"]257            if isinstance(model, (list, tuple)):258                model = model[0]259            model = model.__name__260            if model not in in_table.values():261                missing.append(key)262 263    if len(missing) > 0:264        msg = ", ".join(missing)265        raise ValueError(266            "The following pipeline tags are not present in the `PIPELINE_TAGS_AND_AUTO_MODELS` constant inside "267            f"`utils/update_metadata.py`: {msg}. Please add them!"268        )269 270 271if __name__ == "__main__":272    parser = argparse.ArgumentParser()273    parser.add_argument("--token", type=str, help="The token to use to push to the transformers-metadata dataset.")274    parser.add_argument("--commit_sha", type=str, help="The sha of the commit going with this update.")275    parser.add_argument("--check-only", action="store_true", help="Activate to just check all pipelines are present.")276    args = parser.parse_args()277 278    if args.check_only:279        check_pipeline_tags()280    else:281        update_metadata(args.token, args.commit_sha)282