chendl/compositional_test
1
1# coding=utf-82# Copyright 2020 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import argparse17import glob18import os19import re20 21import black22from doc_builder.style_doc import style_docstrings_in_code23 24from transformers.utils import direct_transformers_import25 26 27# All paths are set with the intent you should run this script from the root of the repo with the command28# python utils/check_copies.py29TRANSFORMERS_PATH = "src/transformers"30PATH_TO_DOCS = "docs/source/en"31REPO_PATH = "."32 33# Mapping for files that are full copies of others (keys are copies, values the file to keep them up to data with)34FULL_COPIES = {35 "examples/tensorflow/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",36 "examples/flax/question-answering/utils_qa.py": "examples/pytorch/question-answering/utils_qa.py",37}38 39 40LOCALIZED_READMES = {41 # If the introduction or the conclusion of the list change, the prompts may need to be updated.42 "README.md": {43 "start_prompt": "🤗 Transformers currently provides the following architectures",44 "end_prompt": "1. Want to contribute a new model?",45 "format_model_list": (46 "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"47 " {paper_authors}.{supplements}"48 ),49 },50 "README_zh-hans.md": {51 "start_prompt": "🤗 Transformers 目前支持如下的架构",52 "end_prompt": "1. 想要贡献新的模型？",53 "format_model_list": (54 "**[{title}]({model_link})** (来自 {paper_affiliations}) 伴随论文 {paper_title_link} 由 {paper_authors}"55 " 发布。{supplements}"56 ),57 },58 "README_zh-hant.md": {59 "start_prompt": "🤗 Transformers 目前支援以下的架構",60 "end_prompt": "1. 想要貢獻新的模型？",61 "format_model_list": (62 "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"63 " {paper_authors}.{supplements}"64 ),65 },66 "README_ko.md": {67 "start_prompt": "🤗 Transformers는 다음 모델들을 제공합니다",68 "end_prompt": "1. 새로운 모델을 올리고 싶나요?",69 "format_model_list": (70 "**[{title}]({model_link})** ({paper_affiliations} 에서 제공)은 {paper_authors}.{supplements}의"71 " {paper_title_link}논문과 함께 발표했습니다."72 ),73 },74 "README_es.md": {75 "start_prompt": "🤗 Transformers actualmente proporciona las siguientes arquitecturas",76 "end_prompt": "1. ¿Quieres aportar un nuevo modelo?",77 "format_model_list": (78 "**[{title}]({model_link})** (from {paper_affiliations}) released with the paper {paper_title_link} by"79 " {paper_authors}.{supplements}"80 ),81 },82 "README_ja.md": {83 "start_prompt": "🤗Transformersは現在、以下のアーキテクチャを提供しています",84 "end_prompt": "1. 新しいモデルを投稿したいですか？",85 "format_model_list": (86 "**[{title}]({model_link})** ({paper_affiliations} から) {paper_authors}.{supplements} から公開された研究論文"87 " {paper_title_link}"88 ),89 },90 "README_hd.md": {91 "start_prompt": "🤗 ट्रांसफॉर्मर वर्तमान में निम्नलिखित आर्किटेक्चर का समर्थन करते हैं",92 "end_prompt": "1. एक नए मॉडल में योगदान देना चाहते हैं?",93 "format_model_list": (94 "**[{title}]({model_link})** ({paper_affiliations} से) {paper_authors}.{supplements} द्वारा"95 "अनुसंधान पत्र {paper_title_link} के साथ जारी किया गया"96 ),97 },98}99 100 101# This is to make sure the transformers module imported is the one in the repo.102transformers_module = direct_transformers_import(TRANSFORMERS_PATH)103 104 105def _should_continue(line, indent):106 return line.startswith(indent) or len(line) <= 1 or re.search(r"^\s*\)(\s*->.*:|:)\s*$", line) is not None107 108 109def find_code_in_transformers(object_name):110 """Find and return the code source code of `object_name`."""111 parts = object_name.split(".")112 i = 0113 114 # First let's find the module where our object lives.115 module = parts[i]116 while i < len(parts) and not os.path.isfile(os.path.join(TRANSFORMERS_PATH, f"{module}.py")):117 i += 1118 if i < len(parts):119 module = os.path.join(module, parts[i])120 if i >= len(parts):121 raise ValueError(122 f"`object_name` should begin with the name of a module of transformers but got {object_name}."123 )124 125 with open(os.path.join(TRANSFORMERS_PATH, f"{module}.py"), "r", encoding="utf-8", newline="\n") as f:126 lines = f.readlines()127 128 # Now let's find the class / func in the code!129 indent = ""130 line_index = 0131 for name in parts[i + 1 :]:132 while (133 line_index < len(lines) and re.search(rf"^{indent}(class|def)\s+{name}(\(|\:)", lines[line_index]) is None134 ):135 line_index += 1136 indent += " "137 line_index += 1138 139 if line_index >= len(lines):140 raise ValueError(f" {object_name} does not match any function or class in {module}.")141 142 # We found the beginning of the class / func, now let's find the end (when the indent diminishes).143 start_index = line_index144 while line_index < len(lines) and _should_continue(lines[line_index], indent):145 line_index += 1146 # Clean up empty lines at the end (if any).147 while len(lines[line_index - 1]) <= 1:148 line_index -= 1149 150 code_lines = lines[start_index:line_index]151 return "".join(code_lines)152 153 154_re_copy_warning = re.compile(r"^(\s*)#\s*Copied from\s+transformers\.(\S+\.\S+)\s*($|\S.*$)")155_re_replace_pattern = re.compile(r"^\s*(\S+)->(\S+)(\s+.*|$)")156_re_fill_pattern = re.compile(r"<FILL\s+[^>]*>")157 158 159def get_indent(code):160 lines = code.split("\n")161 idx = 0162 while idx < len(lines) and len(lines[idx]) == 0:163 idx += 1164 if idx < len(lines):165 return re.search(r"^(\s*)\S", lines[idx]).groups()[0]166 return ""167 168 169def blackify(code):170 """171 Applies the black part of our `make style` command to `code`.172 """173 has_indent = len(get_indent(code)) > 0174 if has_indent:175 code = f"class Bla:\n{code}"176 mode = black.Mode(target_versions={black.TargetVersion.PY37}, line_length=119)177 result = black.format_str(code, mode=mode)178 result, _ = style_docstrings_in_code(result)179 return result[len("class Bla:\n") :] if has_indent else result180 181 182def is_copy_consistent(filename, overwrite=False):183 """184 Check if the code commented as a copy in `filename` matches the original.185 186 Return the differences or overwrites the content depending on `overwrite`.187 """188 with open(filename, "r", encoding="utf-8", newline="\n") as f:189 lines = f.readlines()190 diffs = []191 line_index = 0192 # Not a for loop cause `lines` is going to change (if `overwrite=True`).193 while line_index < len(lines):194 search = _re_copy_warning.search(lines[line_index])195 if search is None:196 line_index += 1197 continue198 199 # There is some copied code here, let's retrieve the original.200 indent, object_name, replace_pattern = search.groups()201 theoretical_code = find_code_in_transformers(object_name)202 theoretical_indent = get_indent(theoretical_code)203 204 start_index = line_index + 1 if indent == theoretical_indent else line_index + 2205 indent = theoretical_indent206 line_index = start_index207 208 # Loop to check the observed code, stop when indentation diminishes or if we see a End copy comment.209 should_continue = True210 while line_index < len(lines) and should_continue:211 line_index += 1212 if line_index >= len(lines):213 break214 line = lines[line_index]215 should_continue = _should_continue(line, indent) and re.search(f"^{indent}# End copy", line) is None216 # Clean up empty lines at the end (if any).217 while len(lines[line_index - 1]) <= 1:218 line_index -= 1219 220 observed_code_lines = lines[start_index:line_index]221 observed_code = "".join(observed_code_lines)222 223 # Before comparing, use the `replace_pattern` on the original code.224 if len(replace_pattern) > 0:225 patterns = replace_pattern.replace("with", "").split(",")226 patterns = [_re_replace_pattern.search(p) for p in patterns]227 for pattern in patterns:228 if pattern is None:229 continue230 obj1, obj2, option = pattern.groups()231 theoretical_code = re.sub(obj1, obj2, theoretical_code)232 if option.strip() == "all-casing":233 theoretical_code = re.sub(obj1.lower(), obj2.lower(), theoretical_code)234 theoretical_code = re.sub(obj1.upper(), obj2.upper(), theoretical_code)235 236 # Blackify after replacement. To be able to do that, we need the header (class or function definition)237 # from the previous line238 theoretical_code = blackify(lines[start_index - 1] + theoretical_code)239 theoretical_code = theoretical_code[len(lines[start_index - 1]) :]240 241 # Test for a diff and act accordingly.242 if observed_code != theoretical_code:243 diff_index = start_index + 1244 for observed_line, theoretical_line in zip(observed_code.split("\n"), theoretical_code.split("\n")):245 if observed_line != theoretical_line:246 break247 diff_index += 1248 diffs.append([object_name, diff_index])249 if overwrite:250 lines = lines[:start_index] + [theoretical_code] + lines[line_index:]251 line_index = start_index + 1252 253 if overwrite and len(diffs) > 0:254 # Warn the user a file has been modified.255 print(f"Detected changes, rewriting {filename}.")256 with open(filename, "w", encoding="utf-8", newline="\n") as f:257 f.writelines(lines)258 return diffs259 260 261def check_copies(overwrite: bool = False):262 all_files = glob.glob(os.path.join(TRANSFORMERS_PATH, "**/*.py"), recursive=True)263 diffs = []264 for filename in all_files:265 new_diffs = is_copy_consistent(filename, overwrite)266 diffs += [f"- {filename}: copy does not match {d[0]} at line {d[1]}" for d in new_diffs]267 if not overwrite and len(diffs) > 0:268 diff = "\n".join(diffs)269 raise Exception(270 "Found the following copy inconsistencies:\n"271 + diff272 + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."273 )274 check_model_list_copy(overwrite=overwrite)275 276 277def check_full_copies(overwrite: bool = False):278 diffs = []279 for target, source in FULL_COPIES.items():280 with open(source, "r", encoding="utf-8") as f:281 source_code = f.read()282 with open(target, "r", encoding="utf-8") as f:283 target_code = f.read()284 if source_code != target_code:285 if overwrite:286 with open(target, "w", encoding="utf-8") as f:287 print(f"Replacing the content of {target} by the one of {source}.")288 f.write(source_code)289 else:290 diffs.append(f"- {target}: copy does not match {source}.")291 292 if not overwrite and len(diffs) > 0:293 diff = "\n".join(diffs)294 raise Exception(295 "Found the following copy inconsistencies:\n"296 + diff297 + "\nRun `make fix-copies` or `python utils/check_copies.py --fix_and_overwrite` to fix them."298 )299 300 301def get_model_list(filename, start_prompt, end_prompt):302 """Extracts the model list from the README."""303 with open(os.path.join(REPO_PATH, filename), "r", encoding="utf-8", newline="\n") as f:304 lines = f.readlines()305 # Find the start of the list.306 start_index = 0307 while not lines[start_index].startswith(start_prompt):308 start_index += 1309 start_index += 1310 311 result = []312 current_line = ""313 end_index = start_index314 315 while not lines[end_index].startswith(end_prompt):316 if lines[end_index].startswith("1."):317 if len(current_line) > 1:318 result.append(current_line)319 current_line = lines[end_index]320 elif len(lines[end_index]) > 1:321 current_line = f"{current_line[:-1]} {lines[end_index].lstrip()}"322 end_index += 1323 if len(current_line) > 1:324 result.append(current_line)325 326 return "".join(result)327 328 329def convert_to_localized_md(model_list, localized_model_list, format_str):330 """Convert `model_list` to each localized README."""331 332 def _rep(match):333 title, model_link, paper_affiliations, paper_title_link, paper_authors, supplements = match.groups()334 return format_str.format(335 title=title,336 model_link=model_link,337 paper_affiliations=paper_affiliations,338 paper_title_link=paper_title_link,339 paper_authors=paper_authors,340 supplements=" " + supplements.strip() if len(supplements) != 0 else "",341 )342 343 # This regex captures metadata from an English model description, including model title, model link,344 # affiliations of the paper, title of the paper, authors of the paper, and supplemental data (see DistilBERT for example).345 _re_capture_meta = re.compile(346 r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\* \(from ([^)]*)\)[^\[]*([^\)]*\)).*?by (.*?[A-Za-z\*]{2,}?)\. (.*)$"347 )348 # This regex is used to synchronize link.349 _re_capture_title_link = re.compile(r"\*\*\[([^\]]*)\]\(([^\)]*)\)\*\*")350 351 if len(localized_model_list) == 0:352 localized_model_index = {}353 else:354 try:355 localized_model_index = {356 re.search(r"\*\*\[([^\]]*)", line).groups()[0]: line357 for line in localized_model_list.strip().split("\n")358 }359 except AttributeError:360 raise AttributeError("A model name in localized READMEs cannot be recognized.")361 362 model_keys = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in model_list.strip().split("\n")]363 364 # We exclude keys in localized README not in the main one.365 readmes_match = not any([k not in model_keys for k in localized_model_index])366 localized_model_index = {k: v for k, v in localized_model_index.items() if k in model_keys}367 368 for model in model_list.strip().split("\n"):369 title, model_link = _re_capture_title_link.search(model).groups()370 if title not in localized_model_index:371 readmes_match = False372 # Add an anchor white space behind a model description string for regex.373 # If metadata cannot be captured, the English version will be directly copied.374 localized_model_index[title] = _re_capture_meta.sub(_rep, model + " ")375 elif _re_fill_pattern.search(localized_model_index[title]) is not None:376 update = _re_capture_meta.sub(_rep, model + " ")377 if update != localized_model_index[title]:378 readmes_match = False379 localized_model_index[title] = update380 else:381 # Synchronize link382 localized_model_index[title] = _re_capture_title_link.sub(383 f"**[{title}]({model_link})**", localized_model_index[title], count=1384 )385 386 sorted_index = sorted(localized_model_index.items(), key=lambda x: x[0].lower())387 388 return readmes_match, "\n".join((x[1] for x in sorted_index)) + "\n"389 390 391def convert_readme_to_index(model_list):392 model_list = model_list.replace("https://huggingface.co/docs/transformers/main/", "")393 return model_list.replace("https://huggingface.co/docs/transformers/", "")394 395 396def _find_text_in_file(filename, start_prompt, end_prompt):397 """398 Find the text in `filename` between a line beginning with `start_prompt` and before `end_prompt`, removing empty399 lines.400 """401 with open(filename, "r", encoding="utf-8", newline="\n") as f:402 lines = f.readlines()403 # Find the start prompt.404 start_index = 0405 while not lines[start_index].startswith(start_prompt):406 start_index += 1407 start_index += 1408 409 end_index = start_index410 while not lines[end_index].startswith(end_prompt):411 end_index += 1412 end_index -= 1413 414 while len(lines[start_index]) <= 1:415 start_index += 1416 while len(lines[end_index]) <= 1:417 end_index -= 1418 end_index += 1419 return "".join(lines[start_index:end_index]), start_index, end_index, lines420 421 422def check_model_list_copy(overwrite=False, max_per_line=119):423 """Check the model lists in the README and index.rst are consistent and maybe `overwrite`."""424 # Fix potential doc links in the README425 with open(os.path.join(REPO_PATH, "README.md"), "r", encoding="utf-8", newline="\n") as f:426 readme = f.read()427 new_readme = readme.replace("https://huggingface.co/transformers", "https://huggingface.co/docs/transformers")428 new_readme = new_readme.replace(429 "https://huggingface.co/docs/main/transformers", "https://huggingface.co/docs/transformers/main"430 )431 if new_readme != readme:432 if overwrite:433 with open(os.path.join(REPO_PATH, "README.md"), "w", encoding="utf-8", newline="\n") as f:434 f.write(new_readme)435 else:436 raise ValueError(437 "The main README contains wrong links to the documentation of Transformers. Run `make fix-copies` to "438 "automatically fix them."439 )440 441 # If the introduction or the conclusion of the list change, the prompts may need to be updated.442 index_list, start_index, end_index, lines = _find_text_in_file(443 filename=os.path.join(PATH_TO_DOCS, "index.mdx"),444 start_prompt="<!--This list is updated automatically from the README",445 end_prompt="### Supported frameworks",446 )447 md_list = get_model_list(448 filename="README.md",449 start_prompt=LOCALIZED_READMES["README.md"]["start_prompt"],450 end_prompt=LOCALIZED_READMES["README.md"]["end_prompt"],451 )452 453 converted_md_lists = []454 for filename, value in LOCALIZED_READMES.items():455 _start_prompt = value["start_prompt"]456 _end_prompt = value["end_prompt"]457 _format_model_list = value["format_model_list"]458 459 localized_md_list = get_model_list(filename, _start_prompt, _end_prompt)460 readmes_match, converted_md_list = convert_to_localized_md(md_list, localized_md_list, _format_model_list)461 462 converted_md_lists.append((filename, readmes_match, converted_md_list, _start_prompt, _end_prompt))463 464 converted_md_list = convert_readme_to_index(md_list)465 if converted_md_list != index_list:466 if overwrite:467 with open(os.path.join(PATH_TO_DOCS, "index.mdx"), "w", encoding="utf-8", newline="\n") as f:468 f.writelines(lines[:start_index] + [converted_md_list] + lines[end_index:])469 else:470 raise ValueError(471 "The model list in the README changed and the list in `index.mdx` has not been updated. Run "472 "`make fix-copies` to fix this."473 )474 475 for converted_md_list in converted_md_lists:476 filename, readmes_match, converted_md, _start_prompt, _end_prompt = converted_md_list477 478 if filename == "README.md":479 continue480 if overwrite:481 _, start_index, end_index, lines = _find_text_in_file(482 filename=os.path.join(REPO_PATH, filename), start_prompt=_start_prompt, end_prompt=_end_prompt483 )484 with open(os.path.join(REPO_PATH, filename), "w", encoding="utf-8", newline="\n") as f:485 f.writelines(lines[:start_index] + [converted_md] + lines[end_index:])486 elif not readmes_match:487 raise ValueError(488 f"The model list in the README changed and the list in `{filename}` has not been updated. Run "489 "`make fix-copies` to fix this."490 )491 492 493SPECIAL_MODEL_NAMES = {494 "Bert Generation": "BERT For Sequence Generation",495 "BigBird": "BigBird-RoBERTa",496 "Data2VecAudio": "Data2Vec",497 "Data2VecText": "Data2Vec",498 "Data2VecVision": "Data2Vec",499 "DonutSwin": "Swin Transformer",500 "Marian": "MarianMT",501 "MaskFormerSwin": "Swin Transformer",502 "OpenAI GPT-2": "GPT-2",503 "OpenAI GPT": "GPT",504 "Perceiver": "Perceiver IO",505 "ViT": "Vision Transformer (ViT)",506}507 508# Update this list with the models that shouldn't be in the README. This only concerns modular models or those who do509# not have an associated paper.510MODELS_NOT_IN_README = [511 "BertJapanese",512 "Encoder decoder",513 "FairSeq Machine-Translation",514 "HerBERT",515 "RetriBERT",516 "Speech Encoder decoder",517 "Speech2Text",518 "Speech2Text2",519 "Vision Encoder decoder",520 "VisionTextDualEncoder",521]522 523 524README_TEMPLATE = (525 "1. **[{model_name}](https://huggingface.co/docs/main/transformers/model_doc/{model_type})** (from "526 "<FILL INSTITUTION>) released with the paper [<FILL PAPER TITLE>](<FILL ARKIV LINK>) by <FILL AUTHORS>."527)528 529 530def check_readme(overwrite=False):531 info = LOCALIZED_READMES["README.md"]532 models, start_index, end_index, lines = _find_text_in_file(533 os.path.join(REPO_PATH, "README.md"),534 info["start_prompt"],535 info["end_prompt"],536 )537 models_in_readme = [re.search(r"\*\*\[([^\]]*)", line).groups()[0] for line in models.strip().split("\n")]538 539 model_names_mapping = transformers_module.models.auto.configuration_auto.MODEL_NAMES_MAPPING540 absents = [541 (key, name)542 for key, name in model_names_mapping.items()543 if SPECIAL_MODEL_NAMES.get(name, name) not in models_in_readme544 ]545 # Remove exceptions546 absents = [(key, name) for key, name in absents if name not in MODELS_NOT_IN_README]547 if len(absents) > 0 and not overwrite:548 print(absents)549 raise ValueError(550 "The main README doesn't contain all models, run `make fix-copies` to fill it with the missing model(s)"551 " then complete the generated entries.\nIf the model is not supposed to be in the main README, add it to"552 " the list `MODELS_NOT_IN_README` in utils/check_copies.py.\nIf it has a different name in the repo than"553 " in the README, map the correspondence in `SPECIAL_MODEL_NAMES` in utils/check_copies.py."554 )555 556 new_models = [README_TEMPLATE.format(model_name=name, model_type=key) for key, name in absents]557 558 all_models = models.strip().split("\n") + new_models559 all_models = sorted(all_models, key=lambda x: re.search(r"\*\*\[([^\]]*)", x).groups()[0].lower())560 all_models = "\n".join(all_models) + "\n"561 562 if all_models != models:563 if overwrite:564 print("Fixing the main README.")565 with open(os.path.join(REPO_PATH, "README.md"), "w", encoding="utf-8", newline="\n") as f:566 f.writelines(lines[:start_index] + [all_models] + lines[end_index:])567 else:568 raise ValueError("The main README model list is not properly sorted. Run `make fix-copies` to fix this.")569 570 571if __name__ == "__main__":572 parser = argparse.ArgumentParser()573 parser.add_argument("--fix_and_overwrite", action="store_true", help="Whether to fix inconsistencies.")574 args = parser.parse_args()575 576 check_readme(args.fix_and_overwrite)577 check_copies(args.fix_and_overwrite)578 check_full_copies(args.fix_and_overwrite)579