chendl/compositional_test

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15"""A script running `create_dummy_models.py` with a pre-defined set of arguments.16 17This file is intended to be used in a CI workflow file without the need of specifying arguments. It creates and uploads18tiny models for all model classes (if their tiny versions are not on the Hub yet), as well as produces an updated19version of `tests/utils/tiny_model_summary.json`. That updated file should be merged into the `main` branch of20`transformers` so the pipeline testing will use the latest created/updated tiny models.21"""22 23 24import argparse25import copy26import json27import multiprocessing28import os29import time30 31from create_dummy_models import COMPOSITE_MODELS, create_tiny_models32from huggingface_hub import ModelFilter, hf_api33 34import transformers35from transformers import AutoFeatureExtractor, AutoImageProcessor, AutoTokenizer36from transformers.image_processing_utils import BaseImageProcessor37 38 39def get_all_model_names():40    model_names = set()41    # Each auto modeling files contains multiple mappings. Let's get them in a dynamic way.42    for module_name in ["modeling_auto", "modeling_tf_auto", "modeling_flax_auto"]:43        module = getattr(transformers.models.auto, module_name, None)44        if module is None:45            continue46        # all mappings in a single auto modeling file47        mapping_names = [48            x49            for x in dir(module)50            if x.endswith("_MAPPING_NAMES")51            and (x.startswith("MODEL_") or x.startswith("TF_MODEL_") or x.startswith("FLAX_MODEL_"))52        ]53        for name in mapping_names:54            mapping = getattr(module, name)55            if mapping is not None:56                for v in mapping.values():57                    if isinstance(v, (list, tuple)):58                        model_names.update(v)59                    elif isinstance(v, str):60                        model_names.add(v)61 62    return sorted(model_names)63 64 65def get_tiny_model_names_from_repo():66    # All model names defined in auto mappings67    model_names = set(get_all_model_names())68 69    with open("tests/utils/tiny_model_summary.json") as fp:70        tiny_model_info = json.load(fp)71    tiny_models_names = set()72    for model_base_name in tiny_model_info:73        tiny_models_names.update(tiny_model_info[model_base_name]["model_classes"])74 75    # Remove a tiny model name if one of its framework implementation hasn't yet a tiny version on the Hub.76    not_on_hub = model_names.difference(tiny_models_names)77    for model_name in copy.copy(tiny_models_names):78        if not model_name.startswith("TF") and f"TF{model_name}" in not_on_hub:79            tiny_models_names.remove(model_name)80        elif model_name.startswith("TF") and model_name[2:] in not_on_hub:81            tiny_models_names.remove(model_name)82 83    return sorted(tiny_models_names)84 85 86def get_tiny_model_summary_from_hub(output_path):87    special_models = COMPOSITE_MODELS.values()88 89    # All tiny model base names on Hub90    model_names = get_all_model_names()91    models = hf_api.list_models(92        filter=ModelFilter(93            author="hf-internal-testing",94        )95    )96    _models = set()97    for x in models:98        model = x.modelId99        org, model = model.split("/")100        if not model.startswith("tiny-random-"):101            continue102        model = model.replace("tiny-random-", "")103        if not model[0].isupper():104            continue105        if model not in model_names and model not in special_models:106            continue107        _models.add(model)108 109    models = sorted(_models)110    # All tiny model names on Hub111    summary = {}112    for model in models:113        repo_id = f"hf-internal-testing/tiny-random-{model}"114        model = model.split("-")[0]115        try:116            repo_info = hf_api.repo_info(repo_id)117            content = {118                "tokenizer_classes": set(),119                "processor_classes": set(),120                "model_classes": set(),121                "sha": repo_info.sha,122            }123        except Exception:124            continue125        try:126            time.sleep(1)127            tokenizer_fast = AutoTokenizer.from_pretrained(repo_id)128            content["tokenizer_classes"].add(tokenizer_fast.__class__.__name__)129        except Exception:130            pass131        try:132            time.sleep(1)133            tokenizer_slow = AutoTokenizer.from_pretrained(repo_id, use_fast=False)134            content["tokenizer_classes"].add(tokenizer_slow.__class__.__name__)135        except Exception:136            pass137        try:138            time.sleep(1)139            img_p = AutoImageProcessor.from_pretrained(repo_id)140            content["processor_classes"].add(img_p.__class__.__name__)141        except Exception:142            pass143        try:144            time.sleep(1)145            feat_p = AutoFeatureExtractor.from_pretrained(repo_id)146            if not isinstance(feat_p, BaseImageProcessor):147                content["processor_classes"].add(feat_p.__class__.__name__)148        except Exception:149            pass150        try:151            time.sleep(1)152            model_class = getattr(transformers, model)153            m = model_class.from_pretrained(repo_id)154            content["model_classes"].add(m.__class__.__name__)155        except Exception:156            pass157        try:158            time.sleep(1)159            model_class = getattr(transformers, f"TF{model}")160            m = model_class.from_pretrained(repo_id)161            content["model_classes"].add(m.__class__.__name__)162        except Exception:163            pass164 165        content["tokenizer_classes"] = sorted(content["tokenizer_classes"])166        content["processor_classes"] = sorted(content["processor_classes"])167        content["model_classes"] = sorted(content["model_classes"])168 169        summary[model] = content170        with open(os.path.join(output_path, "hub_tiny_model_summary.json"), "w") as fp:171            json.dump(summary, fp, ensure_ascii=False, indent=4)172 173 174if __name__ == "__main__":175    parser = argparse.ArgumentParser()176    parser.add_argument("--num_workers", default=1, type=int, help="The number of workers to run.")177    args = parser.parse_args()178 179    # This has to be `spawn` to avoid hanging forever!180    multiprocessing.set_start_method("spawn")181 182    output_path = "tiny_models"183    all = True184    model_types = None185    models_to_skip = get_tiny_model_names_from_repo()186    no_check = True187    upload = True188    organization = "hf-internal-testing"189 190    create_tiny_models(191        output_path,192        all,193        model_types,194        models_to_skip,195        no_check,196        upload,197        organization,198        token=os.environ.get("TOKEN", None),199        num_workers=args.num_workers,200    )201