chendl/compositional_test

1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8#     http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import importlib17import os18import sys19 20 21# This is required to make the module import works (when the python process is running from the root of the repo)22sys.path.append(".")23 24 25r"""26The argument `test_file` in this file refers to a model test file. This should be a string of the from27`tests/models/*/test_modeling_*.py`.28"""29 30 31def get_module_path(test_file):32    """Return the module path of a model test file."""33    components = test_file.split(os.path.sep)34    if components[0:2] != ["tests", "models"]:35        raise ValueError(36            "`test_file` should start with `tests/models/` (with `/` being the OS specific path separator). Got "37            f"{test_file} instead."38        )39    test_fn = components[-1]40    if not test_fn.endswith("py"):41        raise ValueError(f"`test_file` should be a python file. Got {test_fn} instead.")42    if not test_fn.startswith("test_modeling_"):43        raise ValueError(44            f"`test_file` should point to a file name of the form `test_modeling_*.py`. Got {test_fn} instead."45        )46 47    components = components[:-1] + [test_fn.replace(".py", "")]48    test_module_path = ".".join(components)49 50    return test_module_path51 52 53def get_test_module(test_file):54    """Get the module of a model test file."""55    test_module_path = get_module_path(test_file)56    test_module = importlib.import_module(test_module_path)57 58    return test_module59 60 61def get_tester_classes(test_file):62    """Get all classes in a model test file whose names ends with `ModelTester`."""63    tester_classes = []64    test_module = get_test_module(test_file)65    for attr in dir(test_module):66        if attr.endswith("ModelTester"):67            tester_classes.append(getattr(test_module, attr))68 69    # sort with class names70    return sorted(tester_classes, key=lambda x: x.__name__)71 72 73def get_test_classes(test_file):74    """Get all [test] classes in a model test file with attribute `all_model_classes` that are non-empty.75 76    These are usually the (model) test classes containing the (non-slow) tests to run and are subclasses of one of the77    classes `ModelTesterMixin`, `TFModelTesterMixin` or `FlaxModelTesterMixin`, as well as a subclass of78    `unittest.TestCase`. Exceptions include `RagTestMixin` (and its subclasses).79    """80    test_classes = []81    test_module = get_test_module(test_file)82    for attr in dir(test_module):83        attr_value = getattr(test_module, attr)84        # (TF/Flax)ModelTesterMixin is also an attribute in specific model test module. Let's exclude them by checking85        # `all_model_classes` is not empty (which also excludes other special classes).86        model_classes = getattr(attr_value, "all_model_classes", [])87        if len(model_classes) > 0:88            test_classes.append(attr_value)89 90    # sort with class names91    return sorted(test_classes, key=lambda x: x.__name__)92 93 94def get_model_classes(test_file):95    """Get all model classes that appear in `all_model_classes` attributes in a model test file."""96    test_classes = get_test_classes(test_file)97    model_classes = set()98    for test_class in test_classes:99        model_classes.update(test_class.all_model_classes)100 101    # sort with class names102    return sorted(model_classes, key=lambda x: x.__name__)103 104 105def get_model_tester_from_test_class(test_class):106    """Get the model tester class of a model test class."""107    test = test_class()108    if hasattr(test, "setUp"):109        test.setUp()110 111    model_tester = None112    if hasattr(test, "model_tester"):113        # `(TF/Flax)ModelTesterMixin` has this attribute default to `None`. Let's skip this case.114        if test.model_tester is not None:115            model_tester = test.model_tester.__class__116 117    return model_tester118 119 120def get_test_classes_for_model(test_file, model_class):121    """Get all [test] classes in `test_file` that have `model_class` in their `all_model_classes`."""122    test_classes = get_test_classes(test_file)123 124    target_test_classes = []125    for test_class in test_classes:126        if model_class in test_class.all_model_classes:127            target_test_classes.append(test_class)128 129    # sort with class names130    return sorted(target_test_classes, key=lambda x: x.__name__)131 132 133def get_tester_classes_for_model(test_file, model_class):134    """Get all model tester classes in `test_file` that are associated to `model_class`."""135    test_classes = get_test_classes_for_model(test_file, model_class)136 137    tester_classes = []138    for test_class in test_classes:139        tester_class = get_model_tester_from_test_class(test_class)140        if tester_class is not None:141            tester_classes.append(tester_class)142 143    # sort with class names144    return sorted(tester_classes, key=lambda x: x.__name__)145 146 147def get_test_to_tester_mapping(test_file):148    """Get a mapping from [test] classes to model tester classes in `test_file`.149 150    This uses `get_test_classes` which may return classes that are NOT subclasses of `unittest.TestCase`.151    """152    test_classes = get_test_classes(test_file)153    test_tester_mapping = {test_class: get_model_tester_from_test_class(test_class) for test_class in test_classes}154    return test_tester_mapping155 156 157def get_model_to_test_mapping(test_file):158    """Get a mapping from model classes to test classes in `test_file`."""159    model_classes = get_model_classes(test_file)160    model_test_mapping = {161        model_class: get_test_classes_for_model(test_file, model_class) for model_class in model_classes162    }163    return model_test_mapping164 165 166def get_model_to_tester_mapping(test_file):167    """Get a mapping from model classes to model tester classes in `test_file`."""168    model_classes = get_model_classes(test_file)169    model_to_tester_mapping = {170        model_class: get_tester_classes_for_model(test_file, model_class) for model_class in model_classes171    }172    return model_to_tester_mapping173 174 175def to_json(o):176    """Make the information succinct and easy to read.177 178    Avoid the full class representation like `<class 'transformers.models.bert.modeling_bert.BertForMaskedLM'>` when179    displaying the results. Instead, we use class name (`BertForMaskedLM`) for the readability.180    """181    if isinstance(o, str):182        return o183    elif isinstance(o, type):184        return o.__name__185    elif isinstance(o, (list, tuple)):186        return [to_json(x) for x in o]187    elif isinstance(o, dict):188        return {to_json(k): to_json(v) for k, v in o.items()}189    else:190        return o191