chendl/compositional_test

1# coding=utf-82# Copyright 2020 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import inspect17import os18import re19import warnings20from collections import OrderedDict21from difflib import get_close_matches22from pathlib import Path23 24from transformers import is_flax_available, is_tf_available, is_torch_available25from transformers.models.auto import get_values26from transformers.models.auto.configuration_auto import CONFIG_MAPPING_NAMES27from transformers.models.auto.feature_extraction_auto import FEATURE_EXTRACTOR_MAPPING_NAMES28from transformers.models.auto.image_processing_auto import IMAGE_PROCESSOR_MAPPING_NAMES29from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES30from transformers.models.auto.tokenization_auto import TOKENIZER_MAPPING_NAMES31from transformers.utils import ENV_VARS_TRUE_VALUES, direct_transformers_import32 33 34# All paths are set with the intent you should run this script from the root of the repo with the command35# python utils/check_repo.py36PATH_TO_TRANSFORMERS = "src/transformers"37PATH_TO_TESTS = "tests"38PATH_TO_DOC = "docs/source/en"39 40# Update this list with models that are supposed to be private.41PRIVATE_MODELS = [42    "AltRobertaModel",43    "DPRSpanPredictor",44    "LongT5Stack",45    "RealmBertModel",46    "T5Stack",47    "MT5Stack",48    "SwitchTransformersStack",49    "TFDPRSpanPredictor",50    "MaskFormerSwinModel",51    "MaskFormerSwinPreTrainedModel",52    "BridgeTowerTextModel",53    "BridgeTowerVisionModel",54]55 56# Update this list for models that are not tested with a comment explaining the reason it should not be.57# Being in this list is an exception and should **not** be the rule.58IGNORE_NON_TESTED = PRIVATE_MODELS.copy() + [59    # models to ignore for not tested60    "NllbMoeDecoder",61    "NllbMoeEncoder",62    "LlamaDecoder",  # Building part of bigger (tested) model.63    "Blip2QFormerModel",  # Building part of bigger (tested) model.64    "DetaEncoder",  # Building part of bigger (tested) model.65    "DetaDecoder",  # Building part of bigger (tested) model.66    "ErnieMForInformationExtraction",67    "GraphormerEncoder",  # Building part of bigger (tested) model.68    "GraphormerDecoderHead",  # Building part of bigger (tested) model.69    "CLIPSegDecoder",  # Building part of bigger (tested) model.70    "TableTransformerEncoder",  # Building part of bigger (tested) model.71    "TableTransformerDecoder",  # Building part of bigger (tested) model.72    "TimeSeriesTransformerEncoder",  # Building part of bigger (tested) model.73    "TimeSeriesTransformerDecoder",  # Building part of bigger (tested) model.74    "InformerEncoder",  # Building part of bigger (tested) model.75    "InformerDecoder",  # Building part of bigger (tested) model.76    "JukeboxVQVAE",  # Building part of bigger (tested) model.77    "JukeboxPrior",  # Building part of bigger (tested) model.78    "DeformableDetrEncoder",  # Building part of bigger (tested) model.79    "DeformableDetrDecoder",  # Building part of bigger (tested) model.80    "OPTDecoder",  # Building part of bigger (tested) model.81    "FlaxWhisperDecoder",  # Building part of bigger (tested) model.82    "FlaxWhisperEncoder",  # Building part of bigger (tested) model.83    "WhisperDecoder",  # Building part of bigger (tested) model.84    "WhisperEncoder",  # Building part of bigger (tested) model.85    "DecisionTransformerGPT2Model",  # Building part of bigger (tested) model.86    "SegformerDecodeHead",  # Building part of bigger (tested) model.87    "PLBartEncoder",  # Building part of bigger (tested) model.88    "PLBartDecoder",  # Building part of bigger (tested) model.89    "PLBartDecoderWrapper",  # Building part of bigger (tested) model.90    "BigBirdPegasusEncoder",  # Building part of bigger (tested) model.91    "BigBirdPegasusDecoder",  # Building part of bigger (tested) model.92    "BigBirdPegasusDecoderWrapper",  # Building part of bigger (tested) model.93    "DetrEncoder",  # Building part of bigger (tested) model.94    "DetrDecoder",  # Building part of bigger (tested) model.95    "DetrDecoderWrapper",  # Building part of bigger (tested) model.96    "ConditionalDetrEncoder",  # Building part of bigger (tested) model.97    "ConditionalDetrDecoder",  # Building part of bigger (tested) model.98    "M2M100Encoder",  # Building part of bigger (tested) model.99    "M2M100Decoder",  # Building part of bigger (tested) model.100    "MCTCTEncoder",  # Building part of bigger (tested) model.101    "MgpstrModel",  # Building part of bigger (tested) model.102    "Speech2TextEncoder",  # Building part of bigger (tested) model.103    "Speech2TextDecoder",  # Building part of bigger (tested) model.104    "LEDEncoder",  # Building part of bigger (tested) model.105    "LEDDecoder",  # Building part of bigger (tested) model.106    "BartDecoderWrapper",  # Building part of bigger (tested) model.107    "BartEncoder",  # Building part of bigger (tested) model.108    "BertLMHeadModel",  # Needs to be setup as decoder.109    "BlenderbotSmallEncoder",  # Building part of bigger (tested) model.110    "BlenderbotSmallDecoderWrapper",  # Building part of bigger (tested) model.111    "BlenderbotEncoder",  # Building part of bigger (tested) model.112    "BlenderbotDecoderWrapper",  # Building part of bigger (tested) model.113    "MBartEncoder",  # Building part of bigger (tested) model.114    "MBartDecoderWrapper",  # Building part of bigger (tested) model.115    "MegatronBertLMHeadModel",  # Building part of bigger (tested) model.116    "MegatronBertEncoder",  # Building part of bigger (tested) model.117    "MegatronBertDecoder",  # Building part of bigger (tested) model.118    "MegatronBertDecoderWrapper",  # Building part of bigger (tested) model.119    "MvpDecoderWrapper",  # Building part of bigger (tested) model.120    "MvpEncoder",  # Building part of bigger (tested) model.121    "PegasusEncoder",  # Building part of bigger (tested) model.122    "PegasusDecoderWrapper",  # Building part of bigger (tested) model.123    "PegasusXEncoder",  # Building part of bigger (tested) model.124    "PegasusXDecoder",  # Building part of bigger (tested) model.125    "PegasusXDecoderWrapper",  # Building part of bigger (tested) model.126    "DPREncoder",  # Building part of bigger (tested) model.127    "ProphetNetDecoderWrapper",  # Building part of bigger (tested) model.128    "RealmBertModel",  # Building part of bigger (tested) model.129    "RealmReader",  # Not regular model.130    "RealmScorer",  # Not regular model.131    "RealmForOpenQA",  # Not regular model.132    "ReformerForMaskedLM",  # Needs to be setup as decoder.133    "Speech2Text2DecoderWrapper",  # Building part of bigger (tested) model.134    "TFDPREncoder",  # Building part of bigger (tested) model.135    "TFElectraMainLayer",  # Building part of bigger (tested) model (should it be a TFPreTrainedModel ?)136    "TFRobertaForMultipleChoice",  # TODO: fix137    "TFRobertaPreLayerNormForMultipleChoice",  # TODO: fix138    "TrOCRDecoderWrapper",  # Building part of bigger (tested) model.139    "TFWhisperEncoder",  # Building part of bigger (tested) model.140    "TFWhisperDecoder",  # Building part of bigger (tested) model.141    "SeparableConv1D",  # Building part of bigger (tested) model.142    "FlaxBartForCausalLM",  # Building part of bigger (tested) model.143    "FlaxBertForCausalLM",  # Building part of bigger (tested) model. Tested implicitly through FlaxRobertaForCausalLM.144    "OPTDecoderWrapper",145    "TFSegformerDecodeHead",  # Not a regular model.146    "AltRobertaModel",  # Building part of bigger (tested) model.147    "BlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models148    "TFBlipTextLMHeadModel",  # No need to test it as it is tested by BlipTextVision models149    "BridgeTowerTextModel",  # No need to test it as it is tested by BridgeTowerModel model.150    "BridgeTowerVisionModel",  # No need to test it as it is tested by BridgeTowerModel model.151    "SpeechT5Decoder",  # Building part of bigger (tested) model.152    "SpeechT5DecoderWithoutPrenet",  # Building part of bigger (tested) model.153    "SpeechT5DecoderWithSpeechPrenet",  # Building part of bigger (tested) model.154    "SpeechT5DecoderWithTextPrenet",  # Building part of bigger (tested) model.155    "SpeechT5Encoder",  # Building part of bigger (tested) model.156    "SpeechT5EncoderWithoutPrenet",  # Building part of bigger (tested) model.157    "SpeechT5EncoderWithSpeechPrenet",  # Building part of bigger (tested) model.158    "SpeechT5EncoderWithTextPrenet",  # Building part of bigger (tested) model.159    "SpeechT5SpeechDecoder",  # Building part of bigger (tested) model.160    "SpeechT5SpeechEncoder",  # Building part of bigger (tested) model.161    "SpeechT5TextDecoder",  # Building part of bigger (tested) model.162    "SpeechT5TextEncoder",  # Building part of bigger (tested) model.163]164 165# Update this list with test files that don't have a tester with a `all_model_classes` variable and which don't166# trigger the common tests.167TEST_FILES_WITH_NO_COMMON_TESTS = [168    "models/decision_transformer/test_modeling_decision_transformer.py",169    "models/camembert/test_modeling_camembert.py",170    "models/mt5/test_modeling_flax_mt5.py",171    "models/mbart/test_modeling_mbart.py",172    "models/mt5/test_modeling_mt5.py",173    "models/pegasus/test_modeling_pegasus.py",174    "models/camembert/test_modeling_tf_camembert.py",175    "models/mt5/test_modeling_tf_mt5.py",176    "models/xlm_roberta/test_modeling_tf_xlm_roberta.py",177    "models/xlm_roberta/test_modeling_flax_xlm_roberta.py",178    "models/xlm_prophetnet/test_modeling_xlm_prophetnet.py",179    "models/xlm_roberta/test_modeling_xlm_roberta.py",180    "models/vision_text_dual_encoder/test_modeling_vision_text_dual_encoder.py",181    "models/vision_text_dual_encoder/test_modeling_tf_vision_text_dual_encoder.py",182    "models/vision_text_dual_encoder/test_modeling_flax_vision_text_dual_encoder.py",183    "models/decision_transformer/test_modeling_decision_transformer.py",184]185 186# Update this list for models that are not in any of the auto MODEL_XXX_MAPPING. Being in this list is an exception and187# should **not** be the rule.188IGNORE_NON_AUTO_CONFIGURED = PRIVATE_MODELS.copy() + [189    # models to ignore for model xxx mapping190    "AlignTextModel",191    "AlignVisionModel",192    "ClapTextModel",193    "ClapTextModelWithProjection",194    "ClapAudioModel",195    "ClapAudioModelWithProjection",196    "Blip2ForConditionalGeneration",197    "Blip2QFormerModel",198    "Blip2VisionModel",199    "ErnieMForInformationExtraction",200    "GitVisionModel",201    "GraphormerModel",202    "GraphormerForGraphClassification",203    "BlipForConditionalGeneration",204    "BlipForImageTextRetrieval",205    "BlipForQuestionAnswering",206    "BlipVisionModel",207    "BlipTextLMHeadModel",208    "BlipTextModel",209    "TFBlipForConditionalGeneration",210    "TFBlipForImageTextRetrieval",211    "TFBlipForQuestionAnswering",212    "TFBlipVisionModel",213    "TFBlipTextLMHeadModel",214    "TFBlipTextModel",215    "Swin2SRForImageSuperResolution",216    "BridgeTowerForImageAndTextRetrieval",217    "BridgeTowerForMaskedLM",218    "BridgeTowerForContrastiveLearning",219    "CLIPSegForImageSegmentation",220    "CLIPSegVisionModel",221    "CLIPSegTextModel",222    "EsmForProteinFolding",223    "GPTSanJapaneseModel",224    "TimeSeriesTransformerForPrediction",225    "InformerForPrediction",226    "JukeboxVQVAE",227    "JukeboxPrior",228    "PegasusXEncoder",229    "PegasusXDecoder",230    "PegasusXDecoderWrapper",231    "PegasusXEncoder",232    "PegasusXDecoder",233    "PegasusXDecoderWrapper",234    "DPTForDepthEstimation",235    "DecisionTransformerGPT2Model",236    "GLPNForDepthEstimation",237    "ViltForImagesAndTextClassification",238    "ViltForImageAndTextRetrieval",239    "ViltForTokenClassification",240    "ViltForMaskedLM",241    "XGLMEncoder",242    "XGLMDecoder",243    "XGLMDecoderWrapper",244    "PerceiverForMultimodalAutoencoding",245    "PerceiverForOpticalFlow",246    "SegformerDecodeHead",247    "TFSegformerDecodeHead",248    "FlaxBeitForMaskedImageModeling",249    "PLBartEncoder",250    "PLBartDecoder",251    "PLBartDecoderWrapper",252    "BeitForMaskedImageModeling",253    "ChineseCLIPTextModel",254    "ChineseCLIPVisionModel",255    "CLIPTextModel",256    "CLIPTextModelWithProjection",257    "CLIPVisionModel",258    "CLIPVisionModelWithProjection",259    "GroupViTTextModel",260    "GroupViTVisionModel",261    "TFCLIPTextModel",262    "TFCLIPVisionModel",263    "TFGroupViTTextModel",264    "TFGroupViTVisionModel",265    "FlaxCLIPTextModel",266    "FlaxCLIPVisionModel",267    "FlaxWav2Vec2ForCTC",268    "DetrForSegmentation",269    "Pix2StructVisionModel",270    "Pix2StructTextModel",271    "Pix2StructForConditionalGeneration",272    "ConditionalDetrForSegmentation",273    "DPRReader",274    "FlaubertForQuestionAnswering",275    "FlavaImageCodebook",276    "FlavaTextModel",277    "FlavaImageModel",278    "FlavaMultimodalModel",279    "GPT2DoubleHeadsModel",280    "GPTSw3DoubleHeadsModel",281    "LayoutLMForQuestionAnswering",282    "LukeForMaskedLM",283    "LukeForEntityClassification",284    "LukeForEntityPairClassification",285    "LukeForEntitySpanClassification",286    "MgpstrModel",287    "OpenAIGPTDoubleHeadsModel",288    "OwlViTTextModel",289    "OwlViTVisionModel",290    "OwlViTForObjectDetection",291    "RagModel",292    "RagSequenceForGeneration",293    "RagTokenForGeneration",294    "RealmEmbedder",295    "RealmForOpenQA",296    "RealmScorer",297    "RealmReader",298    "TFDPRReader",299    "TFGPT2DoubleHeadsModel",300    "TFLayoutLMForQuestionAnswering",301    "TFOpenAIGPTDoubleHeadsModel",302    "TFRagModel",303    "TFRagSequenceForGeneration",304    "TFRagTokenForGeneration",305    "Wav2Vec2ForCTC",306    "HubertForCTC",307    "SEWForCTC",308    "SEWDForCTC",309    "XLMForQuestionAnswering",310    "XLNetForQuestionAnswering",311    "SeparableConv1D",312    "VisualBertForRegionToPhraseAlignment",313    "VisualBertForVisualReasoning",314    "VisualBertForQuestionAnswering",315    "VisualBertForMultipleChoice",316    "TFWav2Vec2ForCTC",317    "TFHubertForCTC",318    "XCLIPVisionModel",319    "XCLIPTextModel",320    "AltCLIPTextModel",321    "AltCLIPVisionModel",322    "AltRobertaModel",323    "TvltForAudioVisualClassification",324    "SpeechT5ForSpeechToSpeech",325    "SpeechT5ForTextToSpeech",326    "SpeechT5HifiGan",327]328 329# Update this list for models that have multiple model types for the same330# model doc331MODEL_TYPE_TO_DOC_MAPPING = OrderedDict(332    [333        ("data2vec-text", "data2vec"),334        ("data2vec-audio", "data2vec"),335        ("data2vec-vision", "data2vec"),336        ("donut-swin", "donut"),337    ]338)339 340 341# This is to make sure the transformers module imported is the one in the repo.342transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)343 344 345def check_missing_backends():346    missing_backends = []347    if not is_torch_available():348        missing_backends.append("PyTorch")349    if not is_tf_available():350        missing_backends.append("TensorFlow")351    if not is_flax_available():352        missing_backends.append("Flax")353    if len(missing_backends) > 0:354        missing = ", ".join(missing_backends)355        if os.getenv("TRANSFORMERS_IS_CI", "").upper() in ENV_VARS_TRUE_VALUES:356            raise Exception(357                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "358                f"Transformers repo, the following are missing: {missing}."359            )360        else:361            warnings.warn(362                "Full repo consistency checks require all backends to be installed (with `pip install -e .[dev]` in the "363                f"Transformers repo, the following are missing: {missing}. While it's probably fine as long as you "364                "didn't make any change in one of those backends modeling files, you should probably execute the "365                "command above to be on the safe side."366            )367 368 369def check_model_list():370    """Check the model list inside the transformers library."""371    # Get the models from the directory structure of `src/transformers/models/`372    models_dir = os.path.join(PATH_TO_TRANSFORMERS, "models")373    _models = []374    for model in os.listdir(models_dir):375        model_dir = os.path.join(models_dir, model)376        if os.path.isdir(model_dir) and "__init__.py" in os.listdir(model_dir):377            _models.append(model)378 379    # Get the models from the directory structure of `src/transformers/models/`380    models = [model for model in dir(transformers.models) if not model.startswith("__")]381 382    missing_models = sorted(set(_models).difference(models))383    if missing_models:384        raise Exception(385            f"The following models should be included in {models_dir}/__init__.py: {','.join(missing_models)}."386        )387 388 389# If some modeling modules should be ignored for all checks, they should be added in the nested list390# _ignore_modules of this function.391def get_model_modules():392    """Get the model modules inside the transformers library."""393    _ignore_modules = [394        "modeling_auto",395        "modeling_encoder_decoder",396        "modeling_marian",397        "modeling_mmbt",398        "modeling_outputs",399        "modeling_retribert",400        "modeling_utils",401        "modeling_flax_auto",402        "modeling_flax_encoder_decoder",403        "modeling_flax_utils",404        "modeling_speech_encoder_decoder",405        "modeling_flax_speech_encoder_decoder",406        "modeling_flax_vision_encoder_decoder",407        "modeling_transfo_xl_utilities",408        "modeling_tf_auto",409        "modeling_tf_encoder_decoder",410        "modeling_tf_outputs",411        "modeling_tf_pytorch_utils",412        "modeling_tf_utils",413        "modeling_tf_transfo_xl_utilities",414        "modeling_tf_vision_encoder_decoder",415        "modeling_vision_encoder_decoder",416    ]417    modules = []418    for model in dir(transformers.models):419        # There are some magic dunder attributes in the dir, we ignore them420        if not model.startswith("__"):421            model_module = getattr(transformers.models, model)422            for submodule in dir(model_module):423                if submodule.startswith("modeling") and submodule not in _ignore_modules:424                    modeling_module = getattr(model_module, submodule)425                    if inspect.ismodule(modeling_module):426                        modules.append(modeling_module)427    return modules428 429 430def get_models(module, include_pretrained=False):431    """Get the objects in module that are models."""432    models = []433    model_classes = (transformers.PreTrainedModel, transformers.TFPreTrainedModel, transformers.FlaxPreTrainedModel)434    for attr_name in dir(module):435        if not include_pretrained and ("Pretrained" in attr_name or "PreTrained" in attr_name):436            continue437        attr = getattr(module, attr_name)438        if isinstance(attr, type) and issubclass(attr, model_classes) and attr.__module__ == module.__name__:439            models.append((attr_name, attr))440    return models441 442 443def is_a_private_model(model):444    """Returns True if the model should not be in the main init."""445    if model in PRIVATE_MODELS:446        return True447 448    # Wrapper, Encoder and Decoder are all privates449    if model.endswith("Wrapper"):450        return True451    if model.endswith("Encoder"):452        return True453    if model.endswith("Decoder"):454        return True455    if model.endswith("Prenet"):456        return True457    return False458 459 460def check_models_are_in_init():461    """Checks all models defined in the library are in the main init."""462    models_not_in_init = []463    dir_transformers = dir(transformers)464    for module in get_model_modules():465        models_not_in_init += [466            model[0] for model in get_models(module, include_pretrained=True) if model[0] not in dir_transformers467        ]468 469    # Remove private models470    models_not_in_init = [model for model in models_not_in_init if not is_a_private_model(model)]471    if len(models_not_in_init) > 0:472        raise Exception(f"The following models should be in the main init: {','.join(models_not_in_init)}.")473 474 475# If some test_modeling files should be ignored when checking models are all tested, they should be added in the476# nested list _ignore_files of this function.477def get_model_test_files():478    """Get the model test files.479 480    The returned files should NOT contain the `tests` (i.e. `PATH_TO_TESTS` defined in this script). They will be481    considered as paths relative to `tests`. A caller has to use `os.path.join(PATH_TO_TESTS, ...)` to access the files.482    """483 484    _ignore_files = [485        "test_modeling_common",486        "test_modeling_encoder_decoder",487        "test_modeling_flax_encoder_decoder",488        "test_modeling_flax_speech_encoder_decoder",489        "test_modeling_marian",490        "test_modeling_tf_common",491        "test_modeling_tf_encoder_decoder",492    ]493    test_files = []494    # Check both `PATH_TO_TESTS` and `PATH_TO_TESTS/models`495    model_test_root = os.path.join(PATH_TO_TESTS, "models")496    model_test_dirs = []497    for x in os.listdir(model_test_root):498        x = os.path.join(model_test_root, x)499        if os.path.isdir(x):500            model_test_dirs.append(x)501 502    for target_dir in [PATH_TO_TESTS] + model_test_dirs:503        for file_or_dir in os.listdir(target_dir):504            path = os.path.join(target_dir, file_or_dir)505            if os.path.isfile(path):506                filename = os.path.split(path)[-1]507                if "test_modeling" in filename and os.path.splitext(filename)[0] not in _ignore_files:508                    file = os.path.join(*path.split(os.sep)[1:])509                    test_files.append(file)510 511    return test_files512 513 514# This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the tester class515# for the all_model_classes variable.516def find_tested_models(test_file):517    """Parse the content of test_file to detect what's in all_model_classes"""518    # This is a bit hacky but I didn't find a way to import the test_file as a module and read inside the class519    with open(os.path.join(PATH_TO_TESTS, test_file), "r", encoding="utf-8", newline="\n") as f:520        content = f.read()521    all_models = re.findall(r"all_model_classes\s+=\s+\(\s*\(([^\)]*)\)", content)522    # Check with one less parenthesis as well523    all_models += re.findall(r"all_model_classes\s+=\s+\(([^\)]*)\)", content)524    if len(all_models) > 0:525        model_tested = []526        for entry in all_models:527            for line in entry.split(","):528                name = line.strip()529                if len(name) > 0:530                    model_tested.append(name)531        return model_tested532 533 534def check_models_are_tested(module, test_file):535    """Check models defined in module are tested in test_file."""536    # XxxPreTrainedModel are not tested537    defined_models = get_models(module)538    tested_models = find_tested_models(test_file)539    if tested_models is None:540        if test_file.replace(os.path.sep, "/") in TEST_FILES_WITH_NO_COMMON_TESTS:541            return542        return [543            f"{test_file} should define `all_model_classes` to apply common tests to the models it tests. "544            + "If this intentional, add the test filename to `TEST_FILES_WITH_NO_COMMON_TESTS` in the file "545            + "`utils/check_repo.py`."546        ]547    failures = []548    for model_name, _ in defined_models:549        if model_name not in tested_models and model_name not in IGNORE_NON_TESTED:550            failures.append(551                f"{model_name} is defined in {module.__name__} but is not tested in "552                + f"{os.path.join(PATH_TO_TESTS, test_file)}. Add it to the all_model_classes in that file."553                + "If common tests should not applied to that model, add its name to `IGNORE_NON_TESTED`"554                + "in the file `utils/check_repo.py`."555            )556    return failures557 558 559def check_all_models_are_tested():560    """Check all models are properly tested."""561    modules = get_model_modules()562    test_files = get_model_test_files()563    failures = []564    for module in modules:565        test_file = [file for file in test_files if f"test_{module.__name__.split('.')[-1]}.py" in file]566        if len(test_file) == 0:567            failures.append(f"{module.__name__} does not have its corresponding test file {test_file}.")568        elif len(test_file) > 1:569            failures.append(f"{module.__name__} has several test files: {test_file}.")570        else:571            test_file = test_file[0]572            new_failures = check_models_are_tested(module, test_file)573            if new_failures is not None:574                failures += new_failures575    if len(failures) > 0:576        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))577 578 579def get_all_auto_configured_models():580    """Return the list of all models in at least one auto class."""581    result = set()  # To avoid duplicates we concatenate all model classes in a set.582    if is_torch_available():583        for attr_name in dir(transformers.models.auto.modeling_auto):584            if attr_name.startswith("MODEL_") and attr_name.endswith("MAPPING_NAMES"):585                result = result | set(get_values(getattr(transformers.models.auto.modeling_auto, attr_name)))586    if is_tf_available():587        for attr_name in dir(transformers.models.auto.modeling_tf_auto):588            if attr_name.startswith("TF_MODEL_") and attr_name.endswith("MAPPING_NAMES"):589                result = result | set(get_values(getattr(transformers.models.auto.modeling_tf_auto, attr_name)))590    if is_flax_available():591        for attr_name in dir(transformers.models.auto.modeling_flax_auto):592            if attr_name.startswith("FLAX_MODEL_") and attr_name.endswith("MAPPING_NAMES"):593                result = result | set(get_values(getattr(transformers.models.auto.modeling_flax_auto, attr_name)))594    return list(result)595 596 597def ignore_unautoclassed(model_name):598    """Rules to determine if `name` should be in an auto class."""599    # Special white list600    if model_name in IGNORE_NON_AUTO_CONFIGURED:601        return True602    # Encoder and Decoder should be ignored603    if "Encoder" in model_name or "Decoder" in model_name:604        return True605    return False606 607 608def check_models_are_auto_configured(module, all_auto_models):609    """Check models defined in module are each in an auto class."""610    defined_models = get_models(module)611    failures = []612    for model_name, _ in defined_models:613        if model_name not in all_auto_models and not ignore_unautoclassed(model_name):614            failures.append(615                f"{model_name} is defined in {module.__name__} but is not present in any of the auto mapping. "616                "If that is intended behavior, add its name to `IGNORE_NON_AUTO_CONFIGURED` in the file "617                "`utils/check_repo.py`."618            )619    return failures620 621 622def check_all_models_are_auto_configured():623    """Check all models are each in an auto class."""624    check_missing_backends()625    modules = get_model_modules()626    all_auto_models = get_all_auto_configured_models()627    failures = []628    for module in modules:629        new_failures = check_models_are_auto_configured(module, all_auto_models)630        if new_failures is not None:631            failures += new_failures632    if len(failures) > 0:633        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))634 635 636def check_all_auto_object_names_being_defined():637    """Check all names defined in auto (name) mappings exist in the library."""638    check_missing_backends()639 640    failures = []641    mappings_to_check = {642        "TOKENIZER_MAPPING_NAMES": TOKENIZER_MAPPING_NAMES,643        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,644        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,645        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,646    }647 648    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.649    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:650        module = getattr(transformers.models.auto, module_name, None)651        if module is None:652            continue653        # all mappings in a single auto modeling file654        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]655        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})656 657    for name, mapping in mappings_to_check.items():658        for model_type, class_names in mapping.items():659            if not isinstance(class_names, tuple):660                class_names = (class_names,)661                for class_name in class_names:662                    if class_name is None:663                        continue664                    # dummy object is accepted665                    if not hasattr(transformers, class_name):666                        # If the class name is in a model name mapping, let's not check if there is a definition in any modeling667                        # module, if it's a private model defined in this file.668                        if name.endswith("MODEL_MAPPING_NAMES") and is_a_private_model(class_name):669                            continue670                        failures.append(671                            f"`{class_name}` appears in the mapping `{name}` but it is not defined in the library."672                        )673    if len(failures) > 0:674        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))675 676 677def check_all_auto_mapping_names_in_config_mapping_names():678    """Check all keys defined in auto mappings (mappings of names) appear in `CONFIG_MAPPING_NAMES`."""679    check_missing_backends()680 681    failures = []682    # `TOKENIZER_PROCESSOR_MAPPING_NAMES` and `AutoTokenizer` is special, and don't need to follow the rule.683    mappings_to_check = {684        "IMAGE_PROCESSOR_MAPPING_NAMES": IMAGE_PROCESSOR_MAPPING_NAMES,685        "FEATURE_EXTRACTOR_MAPPING_NAMES": FEATURE_EXTRACTOR_MAPPING_NAMES,686        "PROCESSOR_MAPPING_NAMES": PROCESSOR_MAPPING_NAMES,687    }688 689    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.690    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:691        module = getattr(transformers.models.auto, module_name, None)692        if module is None:693            continue694        # all mappings in a single auto modeling file695        mapping_names = [x for x in dir(module) if x.endswith("_MAPPING_NAMES")]696        mappings_to_check.update({name: getattr(module, name) for name in mapping_names})697 698    for name, mapping in mappings_to_check.items():699        for model_type, class_names in mapping.items():700            if model_type not in CONFIG_MAPPING_NAMES:701                failures.append(702                    f"`{model_type}` appears in the mapping `{name}` but it is not defined in the keys of "703                    "`CONFIG_MAPPING_NAMES`."704                )705    if len(failures) > 0:706        raise Exception(f"There were {len(failures)} failures:\n" + "\n".join(failures))707 708 709_re_decorator = re.compile(r"^\s*@(\S+)\s+$")710 711 712def check_decorator_order(filename):713    """Check that in the test file `filename` the slow decorator is always last."""714    with open(filename, "r", encoding="utf-8", newline="\n") as f:715        lines = f.readlines()716    decorator_before = None717    errors = []718    for i, line in enumerate(lines):719        search = _re_decorator.search(line)720        if search is not None:721            decorator_name = search.groups()[0]722            if decorator_before is not None and decorator_name.startswith("parameterized"):723                errors.append(i)724            decorator_before = decorator_name725        elif decorator_before is not None:726            decorator_before = None727    return errors728 729 730def check_all_decorator_order():731    """Check that in all test files, the slow decorator is always last."""732    errors = []733    for fname in os.listdir(PATH_TO_TESTS):734        if fname.endswith(".py"):735            filename = os.path.join(PATH_TO_TESTS, fname)736            new_errors = check_decorator_order(filename)737            errors += [f"- {filename}, line {i}" for i in new_errors]738    if len(errors) > 0:739        msg = "\n".join(errors)740        raise ValueError(741            "The parameterized decorator (and its variants) should always be first, but this is not the case in the"742            f" following files:\n{msg}"743        )744 745 746def find_all_documented_objects():747    """Parse the content of all doc files to detect which classes and functions it documents"""748    documented_obj = []749    for doc_file in Path(PATH_TO_DOC).glob("**/*.rst"):750        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:751            content = f.read()752        raw_doc_objs = re.findall(r"(?:autoclass|autofunction):: transformers.(\S+)\s+", content)753        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]754    for doc_file in Path(PATH_TO_DOC).glob("**/*.mdx"):755        with open(doc_file, "r", encoding="utf-8", newline="\n") as f:756            content = f.read()757        raw_doc_objs = re.findall("\[\[autodoc\]\]\s+(\S+)\s+", content)758        documented_obj += [obj.split(".")[-1] for obj in raw_doc_objs]759    return documented_obj760 761 762# One good reason for not being documented is to be deprecated. Put in this list deprecated objects.763DEPRECATED_OBJECTS = [764    "AutoModelWithLMHead",765    "BartPretrainedModel",766    "DataCollator",767    "DataCollatorForSOP",768    "GlueDataset",769    "GlueDataTrainingArguments",770    "LineByLineTextDataset",771    "LineByLineWithRefDataset",772    "LineByLineWithSOPTextDataset",773    "PretrainedBartModel",774    "PretrainedFSMTModel",775    "SingleSentenceClassificationProcessor",776    "SquadDataTrainingArguments",777    "SquadDataset",778    "SquadExample",779    "SquadFeatures",780    "SquadV1Processor",781    "SquadV2Processor",782    "TFAutoModelWithLMHead",783    "TFBartPretrainedModel",784    "TextDataset",785    "TextDatasetForNextSentencePrediction",786    "Wav2Vec2ForMaskedLM",787    "Wav2Vec2Tokenizer",788    "glue_compute_metrics",789    "glue_convert_examples_to_features",790    "glue_output_modes",791    "glue_processors",792    "glue_tasks_num_labels",793    "squad_convert_examples_to_features",794    "xnli_compute_metrics",795    "xnli_output_modes",796    "xnli_processors",797    "xnli_tasks_num_labels",798    "TFTrainer",799    "TFTrainingArguments",800]801 802# Exceptionally, some objects should not be documented after all rules passed.803# ONLY PUT SOMETHING IN THIS LIST AS A LAST RESORT!804UNDOCUMENTED_OBJECTS = [805    "AddedToken",  # This is a tokenizers class.806    "BasicTokenizer",  # Internal, should never have been in the main init.807    "CharacterTokenizer",  # Internal, should never have been in the main init.808    "DPRPretrainedReader",  # Like an Encoder.809    "DummyObject",  # Just picked by mistake sometimes.810    "MecabTokenizer",  # Internal, should never have been in the main init.811    "ModelCard",  # Internal type.812    "SqueezeBertModule",  # Internal building block (should have been called SqueezeBertLayer)813    "TFDPRPretrainedReader",  # Like an Encoder.814    "TransfoXLCorpus",  # Internal type.815    "WordpieceTokenizer",  # Internal, should never have been in the main init.816    "absl",  # External module817    "add_end_docstrings",  # Internal, should never have been in the main init.818    "add_start_docstrings",  # Internal, should never have been in the main init.819    "convert_tf_weight_name_to_pt_weight_name",  # Internal used to convert model weights820    "logger",  # Internal logger821    "logging",  # External module822    "requires_backends",  # Internal function823    "AltRobertaModel",  # Internal module824]825 826# This list should be empty. Objects in it should get their own doc page.827SHOULD_HAVE_THEIR_OWN_PAGE = [828    # Benchmarks829    "PyTorchBenchmark",830    "PyTorchBenchmarkArguments",831    "TensorFlowBenchmark",832    "TensorFlowBenchmarkArguments",833    "AutoBackbone",834    "BitBackbone",835    "ConvNextBackbone",836    "ConvNextV2Backbone",837    "DinatBackbone",838    "MaskFormerSwinBackbone",839    "MaskFormerSwinConfig",840    "MaskFormerSwinModel",841    "NatBackbone",842    "ResNetBackbone",843    "SwinBackbone",844]845 846 847def ignore_undocumented(name):848    """Rules to determine if `name` should be undocumented."""849    # NOT DOCUMENTED ON PURPOSE.850    # Constants uppercase are not documented.851    if name.isupper():852        return True853    # PreTrainedModels / Encoders / Decoders / Layers / Embeddings / Attention are not documented.854    if (855        name.endswith("PreTrainedModel")856        or name.endswith("Decoder")857        or name.endswith("Encoder")858        or name.endswith("Layer")859        or name.endswith("Embeddings")860        or name.endswith("Attention")861    ):862        return True863    # Submodules are not documented.864    if os.path.isdir(os.path.join(PATH_TO_TRANSFORMERS, name)) or os.path.isfile(865        os.path.join(PATH_TO_TRANSFORMERS, f"{name}.py")866    ):867        return True868    # All load functions are not documented.869    if name.startswith("load_tf") or name.startswith("load_pytorch"):870        return True871    # is_xxx_available functions are not documented.872    if name.startswith("is_") and name.endswith("_available"):873        return True874    # Deprecated objects are not documented.875    if name in DEPRECATED_OBJECTS or name in UNDOCUMENTED_OBJECTS:876        return True877    # MMBT model does not really work.878    if name.startswith("MMBT"):879        return True880    if name in SHOULD_HAVE_THEIR_OWN_PAGE:881        return True882    return False883 884 885def check_all_objects_are_documented():886    """Check all models are properly documented."""887    documented_objs = find_all_documented_objects()888    modules = transformers._modules889    objects = [c for c in dir(transformers) if c not in modules and not c.startswith("_")]890    undocumented_objs = [c for c in objects if c not in documented_objs and not ignore_undocumented(c)]891    if len(undocumented_objs) > 0:892        raise Exception(893            "The following objects are in the public init so should be documented:\n - "894            + "\n - ".join(undocumented_objs)895        )896    check_docstrings_are_in_md()897    check_model_type_doc_match()898 899 900def check_model_type_doc_match():901    """Check all doc pages have a corresponding model type."""902    model_doc_folder = Path(PATH_TO_DOC) / "model_doc"903    model_docs = [m.stem for m in model_doc_folder.glob("*.mdx")]904 905    model_types = list(transformers.models.auto.configuration_auto.MODEL_NAMES_MAPPING.keys())906    model_types = [MODEL_TYPE_TO_DOC_MAPPING[m] if m in MODEL_TYPE_TO_DOC_MAPPING else m for m in model_types]907 908    errors = []909    for m in model_docs:910        if m not in model_types and m != "auto":911            close_matches = get_close_matches(m, model_types)912            error_message = f"{m} is not a proper model identifier."913            if len(close_matches) > 0:914                close_matches = "/".join(close_matches)915                error_message += f" Did you mean {close_matches}?"916            errors.append(error_message)917 918    if len(errors) > 0:919        raise ValueError(920            "Some model doc pages do not match any existing model type:\n"921            + "\n".join(errors)922            + "\nYou can add any missing model type to the `MODEL_NAMES_MAPPING` constant in "923            "models/auto/configuration_auto.py."924        )925 926 927# Re pattern to catch :obj:`xx`, :class:`xx`, :func:`xx` or :meth:`xx`.928_re_rst_special_words = re.compile(r":(?:obj|func|class|meth):`([^`]+)`")929# Re pattern to catch things between double backquotes.930_re_double_backquotes = re.compile(r"(^|[^`])``([^`]+)``([^`]|$)")931# Re pattern to catch example introduction.932_re_rst_example = re.compile(r"^\s*Example.*::\s*$", flags=re.MULTILINE)933 934 935def is_rst_docstring(docstring):936    """937    Returns `True` if `docstring` is written in rst.938    """939    if _re_rst_special_words.search(docstring) is not None:940        return True941    if _re_double_backquotes.search(docstring) is not None:942        return True943    if _re_rst_example.search(docstring) is not None:944        return True945    return False946 947 948def check_docstrings_are_in_md():949    """Check all docstrings are in md"""950    files_with_rst = []951    for file in Path(PATH_TO_TRANSFORMERS).glob("**/*.py"):952        with open(file, encoding="utf-8") as f:953            code = f.read()954        docstrings = code.split('"""')955 956        for idx, docstring in enumerate(docstrings):957            if idx % 2 == 0 or not is_rst_docstring(docstring):958                continue959            files_with_rst.append(file)960            break961 962    if len(files_with_rst) > 0:963        raise ValueError(964            "The following files have docstrings written in rst:\n"965            + "\n".join([f"- {f}" for f in files_with_rst])966            + "\nTo fix this run `doc-builder convert path_to_py_file` after installing `doc-builder`\n"967            "(`pip install git+https://github.com/huggingface/doc-builder`)"968        )969 970 971def check_repo_quality():972    """Check all models are properly tested and documented."""973    print("Checking all models are included.")974    check_model_list()975    print("Checking all models are public.")976    check_models_are_in_init()977    print("Checking all models are properly tested.")978    check_all_decorator_order()979    check_all_models_are_tested()980    print("Checking all objects are properly documented.")981    check_all_objects_are_documented()982    print("Checking all models are in at least one auto class.")983    check_all_models_are_auto_configured()984    print("Checking all names in auto name mappings are defined.")985    check_all_auto_object_names_being_defined()986    print("Checking all keys in auto name mappings are defined in `CONFIG_MAPPING_NAMES`.")987    check_all_auto_mapping_names_in_config_mapping_names()988 989 990if __name__ == "__main__":991    check_repo_quality()992