chendl/compositional_test
1
1# coding=utf-82# Copyright 2020 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import collections17import os18import re19from pathlib import Path20 21 22PATH_TO_TRANSFORMERS = "src/transformers"23 24 25# Matches is_xxx_available()26_re_backend = re.compile(r"is\_([a-z_]*)_available()")27# Catches a one-line _import_struct = {xxx}28_re_one_line_import_struct = re.compile(r"^_import_structure\s+=\s+\{([^\}]+)\}")29# Catches a line with a key-values pattern: "bla": ["foo", "bar"]30_re_import_struct_key_value = re.compile(r'\s+"\S*":\s+\[([^\]]*)\]')31# Catches a line if not is_foo_available32_re_test_backend = re.compile(r"^\s*if\s+not\s+is\_[a-z_]*\_available\(\)")33# Catches a line _import_struct["bla"].append("foo")34_re_import_struct_add_one = re.compile(r'^\s*_import_structure\["\S*"\]\.append\("(\S*)"\)')35# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]36_re_import_struct_add_many = re.compile(r"^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]")37# Catches a line with an object between quotes and a comma: "MyModel",38_re_quote_object = re.compile('^\s+"([^"]+)",')39# Catches a line with objects between brackets only: ["foo", "bar"],40_re_between_brackets = re.compile("^\s+\[([^\]]+)\]")41# Catches a line with from foo import bar, bla, boo42_re_import = re.compile(r"\s+from\s+\S*\s+import\s+([^\(\s].*)\n")43# Catches a line with try:44_re_try = re.compile(r"^\s*try:")45# Catches a line with else:46_re_else = re.compile(r"^\s*else:")47 48 49def find_backend(line):50 """Find one (or multiple) backend in a code line of the init."""51 if _re_test_backend.search(line) is None:52 return None53 backends = [b[0] for b in _re_backend.findall(line)]54 backends.sort()55 return "_and_".join(backends)56 57 58def parse_init(init_file):59 """60 Read an init_file and parse (per backend) the _import_structure objects defined and the TYPE_CHECKING objects61 defined62 """63 with open(init_file, "r", encoding="utf-8", newline="\n") as f:64 lines = f.readlines()65 66 line_index = 067 while line_index < len(lines) and not lines[line_index].startswith("_import_structure = {"):68 line_index += 169 70 # If this is a traditional init, just return.71 if line_index >= len(lines):72 return None73 74 # First grab the objects without a specific backend in _import_structure75 objects = []76 while not lines[line_index].startswith("if TYPE_CHECKING") and find_backend(lines[line_index]) is None:77 line = lines[line_index]78 # If we have everything on a single line, let's deal with it.79 if _re_one_line_import_struct.search(line):80 content = _re_one_line_import_struct.search(line).groups()[0]81 imports = re.findall("\[([^\]]+)\]", content)82 for imp in imports:83 objects.extend([obj[1:-1] for obj in imp.split(", ")])84 line_index += 185 continue86 single_line_import_search = _re_import_struct_key_value.search(line)87 if single_line_import_search is not None:88 imports = [obj[1:-1] for obj in single_line_import_search.groups()[0].split(", ") if len(obj) > 0]89 objects.extend(imports)90 elif line.startswith(" " * 8 + '"'):91 objects.append(line[9:-3])92 line_index += 193 94 import_dict_objects = {"none": objects}95 # Let's continue with backend-specific objects in _import_structure96 while not lines[line_index].startswith("if TYPE_CHECKING"):97 # If the line is an if not is_backend_available, we grab all objects associated.98 backend = find_backend(lines[line_index])99 # Check if the backend declaration is inside a try block:100 if _re_try.search(lines[line_index - 1]) is None:101 backend = None102 103 if backend is not None:104 line_index += 1105 106 # Scroll until we hit the else block of try-except-else107 while _re_else.search(lines[line_index]) is None:108 line_index += 1109 110 line_index += 1111 112 objects = []113 # Until we unindent, add backend objects to the list114 while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 4):115 line = lines[line_index]116 if _re_import_struct_add_one.search(line) is not None:117 objects.append(_re_import_struct_add_one.search(line).groups()[0])118 elif _re_import_struct_add_many.search(line) is not None:119 imports = _re_import_struct_add_many.search(line).groups()[0].split(", ")120 imports = [obj[1:-1] for obj in imports if len(obj) > 0]121 objects.extend(imports)122 elif _re_between_brackets.search(line) is not None:123 imports = _re_between_brackets.search(line).groups()[0].split(", ")124 imports = [obj[1:-1] for obj in imports if len(obj) > 0]125 objects.extend(imports)126 elif _re_quote_object.search(line) is not None:127 objects.append(_re_quote_object.search(line).groups()[0])128 elif line.startswith(" " * 8 + '"'):129 objects.append(line[9:-3])130 elif line.startswith(" " * 12 + '"'):131 objects.append(line[13:-3])132 line_index += 1133 134 import_dict_objects[backend] = objects135 else:136 line_index += 1137 138 # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend139 objects = []140 while (141 line_index < len(lines)142 and find_backend(lines[line_index]) is None143 and not lines[line_index].startswith("else")144 ):145 line = lines[line_index]146 single_line_import_search = _re_import.search(line)147 if single_line_import_search is not None:148 objects.extend(single_line_import_search.groups()[0].split(", "))149 elif line.startswith(" " * 8):150 objects.append(line[8:-2])151 line_index += 1152 153 type_hint_objects = {"none": objects}154 # Let's continue with backend-specific objects155 while line_index < len(lines):156 # If the line is an if is_backend_available, we grab all objects associated.157 backend = find_backend(lines[line_index])158 # Check if the backend declaration is inside a try block:159 if _re_try.search(lines[line_index - 1]) is None:160 backend = None161 162 if backend is not None:163 line_index += 1164 165 # Scroll until we hit the else block of try-except-else166 while _re_else.search(lines[line_index]) is None:167 line_index += 1168 169 line_index += 1170 171 objects = []172 # Until we unindent, add backend objects to the list173 while len(lines[line_index]) <= 1 or lines[line_index].startswith(" " * 8):174 line = lines[line_index]175 single_line_import_search = _re_import.search(line)176 if single_line_import_search is not None:177 objects.extend(single_line_import_search.groups()[0].split(", "))178 elif line.startswith(" " * 12):179 objects.append(line[12:-2])180 line_index += 1181 182 type_hint_objects[backend] = objects183 else:184 line_index += 1185 186 return import_dict_objects, type_hint_objects187 188 189def analyze_results(import_dict_objects, type_hint_objects):190 """191 Analyze the differences between _import_structure objects and TYPE_CHECKING objects found in an init.192 """193 194 def find_duplicates(seq):195 return [k for k, v in collections.Counter(seq).items() if v > 1]196 197 if list(import_dict_objects.keys()) != list(type_hint_objects.keys()):198 return ["Both sides of the init do not have the same backends!"]199 200 errors = []201 for key in import_dict_objects.keys():202 duplicate_imports = find_duplicates(import_dict_objects[key])203 if duplicate_imports:204 errors.append(f"Duplicate _import_structure definitions for: {duplicate_imports}")205 duplicate_type_hints = find_duplicates(type_hint_objects[key])206 if duplicate_type_hints:207 errors.append(f"Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}")208 209 if sorted(set(import_dict_objects[key])) != sorted(set(type_hint_objects[key])):210 name = "base imports" if key == "none" else f"{key} backend"211 errors.append(f"Differences for {name}:")212 for a in type_hint_objects[key]:213 if a not in import_dict_objects[key]:214 errors.append(f" {a} in TYPE_HINT but not in _import_structure.")215 for a in import_dict_objects[key]:216 if a not in type_hint_objects[key]:217 errors.append(f" {a} in _import_structure but not in TYPE_HINT.")218 return errors219 220 221def check_all_inits():222 """223 Check all inits in the transformers repo and raise an error if at least one does not define the same objects in224 both halves.225 """226 failures = []227 for root, _, files in os.walk(PATH_TO_TRANSFORMERS):228 if "__init__.py" in files:229 fname = os.path.join(root, "__init__.py")230 objects = parse_init(fname)231 if objects is not None:232 errors = analyze_results(*objects)233 if len(errors) > 0:234 errors[0] = f"Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"235 failures.append("\n".join(errors))236 if len(failures) > 0:237 raise ValueError("\n\n".join(failures))238 239 240def get_transformers_submodules():241 """242 Returns the list of Transformers submodules.243 """244 submodules = []245 for path, directories, files in os.walk(PATH_TO_TRANSFORMERS):246 for folder in directories:247 # Ignore private modules248 if folder.startswith("_"):249 directories.remove(folder)250 continue251 # Ignore leftovers from branches (empty folders apart from pycache)252 if len(list((Path(path) / folder).glob("*.py"))) == 0:253 continue254 short_path = str((Path(path) / folder).relative_to(PATH_TO_TRANSFORMERS))255 submodule = short_path.replace(os.path.sep, ".")256 submodules.append(submodule)257 for fname in files:258 if fname == "__init__.py":259 continue260 short_path = str((Path(path) / fname).relative_to(PATH_TO_TRANSFORMERS))261 submodule = short_path.replace(".py", "").replace(os.path.sep, ".")262 if len(submodule.split(".")) == 1:263 submodules.append(submodule)264 return submodules265 266 267IGNORE_SUBMODULES = [268 "convert_pytorch_checkpoint_to_tf2",269 "modeling_flax_pytorch_utils",270 "models.esm.openfold_utils",271]272 273 274def check_submodules():275 # This is to make sure the transformers module imported is the one in the repo.276 from transformers.utils import direct_transformers_import277 278 transformers = direct_transformers_import(PATH_TO_TRANSFORMERS)279 280 import_structure_keys = set(transformers._import_structure.keys())281 # This contains all the base keys of the _import_structure object defined in the init, but if the user is missing282 # some optional dependencies, they may not have all of them. Thus we read the init to read all additions and283 # (potentiall re-) add them.284 with open(os.path.join(PATH_TO_TRANSFORMERS, "__init__.py"), "r") as f:285 init_content = f.read()286 import_structure_keys.update(set(re.findall(r"import_structure\[\"([^\"]*)\"\]", init_content)))287 288 module_not_registered = [289 module290 for module in get_transformers_submodules()291 if module not in IGNORE_SUBMODULES and module not in import_structure_keys292 ]293 294 if len(module_not_registered) > 0:295 list_of_modules = "\n".join(f"- {module}" for module in module_not_registered)296 raise ValueError(297 "The following submodules are not properly registed in the main init of Transformers:\n"298 f"{list_of_modules}\n"299 "Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value."300 )301 302 303if __name__ == "__main__":304 check_all_inits()305 check_submodules()306