chendl/compositional_test
1
1# Copyright 2022 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import os16import shutil17import sys18import tempfile19import unittest20from contextlib import contextmanager21from pathlib import Path22 23from git import Repo24 25from transformers.testing_utils import CaptureStdout26 27 28REPO_PATH = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))29sys.path.append(os.path.join(REPO_PATH, "utils"))30 31import tests_fetcher # noqa: E40232from tests_fetcher import ( # noqa: E40233 checkout_commit,34 clean_code,35 create_module_to_test_map,36 create_reverse_dependency_map,37 create_reverse_dependency_tree,38 diff_is_docstring_only,39 extract_imports,40 get_all_tests,41 get_diff,42 get_module_dependencies,43 get_tree_starting_at,44 infer_tests_to_run,45 parse_commit_message,46 print_tree_deps_of,47)48 49 50BERT_MODELING_FILE = "src/transformers/models/bert/modeling_bert.py"51BERT_MODEL_FILE = """from ...modeling_utils import PreTrainedModel52from ...utils import is_torch_available53from .configuration_bert import BertConfig54 55class BertModel:56 '''57 This is the docstring.58 '''59 This is the code60"""61 62BERT_MODEL_FILE_NEW_DOCSTRING = """from ...modeling_utils import PreTrainedModel63from ...utils import is_torch_available64from .configuration_bert import BertConfig65 66class BertModel:67 '''68 This is the docstring. It has been updated.69 '''70 This is the code71"""72 73BERT_MODEL_FILE_NEW_CODE = """from ...modeling_utils import PreTrainedModel74from ...utils import is_torch_available75from .configuration_bert import BertConfig76 77class BertModel:78 '''79 This is the docstring.80 '''81 This is the code. It has been updated82"""83 84 85def create_tmp_repo(tmp_dir, models=None):86 """87 Creates a repository in a temporary directory mimicking the structure of Transformers. Uses the list of models88 provided (which defaults to just `["bert"]`).89 """90 tmp_dir = Path(tmp_dir)91 if tmp_dir.exists():92 shutil.rmtree(tmp_dir)93 tmp_dir.mkdir(exist_ok=True)94 repo = Repo.init(tmp_dir)95 96 if models is None:97 models = ["bert"]98 class_names = [model[0].upper() + model[1:] for model in models]99 100 transformers_dir = tmp_dir / "src" / "transformers"101 transformers_dir.mkdir(parents=True, exist_ok=True)102 with open(transformers_dir / "__init__.py", "w") as f:103 init_lines = ["from .utils import cached_file, is_torch_available"]104 init_lines.extend(105 [f"from .models.{model} import {cls}Config, {cls}Model" for model, cls in zip(models, class_names)]106 )107 f.write("\n".join(init_lines) + "\n")108 with open(transformers_dir / "configuration_utils.py", "w") as f:109 f.write("from .utils import cached_file\n\ncode")110 with open(transformers_dir / "modeling_utils.py", "w") as f:111 f.write("from .utils import cached_file\n\ncode")112 113 utils_dir = tmp_dir / "src" / "transformers" / "utils"114 utils_dir.mkdir(exist_ok=True)115 with open(utils_dir / "__init__.py", "w") as f:116 f.write("from .hub import cached_file\nfrom .imports import is_torch_available\n")117 with open(utils_dir / "hub.py", "w") as f:118 f.write("import huggingface_hub\n\ncode")119 with open(utils_dir / "imports.py", "w") as f:120 f.write("code")121 122 model_dir = tmp_dir / "src" / "transformers" / "models"123 model_dir.mkdir(parents=True, exist_ok=True)124 with open(model_dir / "__init__.py", "w") as f:125 f.write("\n".join([f"import {model}" for model in models]))126 127 for model, cls in zip(models, class_names):128 model_dir = tmp_dir / "src" / "transformers" / "models" / model129 model_dir.mkdir(parents=True, exist_ok=True)130 with open(model_dir / "__init__.py", "w") as f:131 f.write(f"from .configuration_{model} import {cls}Config\nfrom .modeling_{model} import {cls}Model\n")132 with open(model_dir / f"configuration_{model}.py", "w") as f:133 f.write("from ...configuration_utils import PretrainedConfig\ncode")134 with open(model_dir / f"modeling_{model}.py", "w") as f:135 modeling_code = BERT_MODEL_FILE.replace("bert", model).replace("Bert", cls)136 f.write(modeling_code)137 138 test_dir = tmp_dir / "tests"139 test_dir.mkdir(exist_ok=True)140 with open(test_dir / "test_modeling_common.py", "w") as f:141 f.write("from transformers.modeling_utils import PreTrainedModel\ncode")142 143 for model, cls in zip(models, class_names):144 test_model_dir = test_dir / "models" / model145 test_model_dir.mkdir(parents=True, exist_ok=True)146 (test_model_dir / "__init__.py").touch()147 with open(test_model_dir / f"test_modeling_{model}.py", "w") as f:148 f.write(149 f"from transformers import {cls}Config, {cls}Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode"150 )151 152 repo.index.add(["src", "tests"])153 repo.index.commit("Initial commit")154 repo.create_head("main")155 repo.head.reference = repo.refs.main156 repo.delete_head("master")157 return repo158 159 160@contextmanager161def patch_transformer_repo_path(new_folder):162 """163 Temporarily patches the variables defines in `tests_fetcher` to use a different location for the repo.164 """165 old_repo_path = tests_fetcher.PATH_TO_REPO166 tests_fetcher.PATH_TO_REPO = Path(new_folder).resolve()167 tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"168 tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"169 try:170 yield171 finally:172 tests_fetcher.PATH_TO_REPO = old_repo_path173 tests_fetcher.PATH_TO_TRANFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"174 tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"175 176 177def commit_changes(filenames, contents, repo, commit_message="Commit"):178 """179 Commit new `contents` to `filenames` inside a given `repo`.180 """181 if not isinstance(filenames, list):182 filenames = [filenames]183 if not isinstance(contents, list):184 contents = [contents]185 186 folder = Path(repo.working_dir)187 for filename, content in zip(filenames, contents):188 with open(folder / filename, "w") as f:189 f.write(content)190 repo.index.add(filenames)191 commit = repo.index.commit(commit_message)192 return commit.hexsha193 194 195class TestFetcherTester(unittest.TestCase):196 def test_checkout_commit(self):197 with tempfile.TemporaryDirectory() as tmp_folder:198 tmp_folder = Path(tmp_folder)199 repo = create_tmp_repo(tmp_folder)200 initial_sha = repo.head.commit.hexsha201 new_sha = commit_changes(BERT_MODELING_FILE, BERT_MODEL_FILE_NEW_DOCSTRING, repo)202 203 assert repo.head.commit.hexsha == new_sha204 with checkout_commit(repo, initial_sha):205 assert repo.head.commit.hexsha == initial_sha206 with open(tmp_folder / BERT_MODELING_FILE) as f:207 assert f.read() == BERT_MODEL_FILE208 209 assert repo.head.commit.hexsha == new_sha210 with open(tmp_folder / BERT_MODELING_FILE) as f:211 assert f.read() == BERT_MODEL_FILE_NEW_DOCSTRING212 213 def test_clean_code(self):214 # Clean code removes all strings in triple quotes215 assert clean_code('"""\nDocstring\n"""\ncode\n"""Long string"""\ncode\n') == "code\ncode"216 assert clean_code("'''\nDocstring\n'''\ncode\n'''Long string'''\ncode\n'''") == "code\ncode"217 218 # Clean code removes all comments219 assert clean_code("code\n# Comment\ncode") == "code\ncode"220 assert clean_code("code # inline comment\ncode") == "code \ncode"221 222 def test_get_all_tests(self):223 with tempfile.TemporaryDirectory() as tmp_folder:224 tmp_folder = Path(tmp_folder)225 create_tmp_repo(tmp_folder)226 with patch_transformer_repo_path(tmp_folder):227 assert get_all_tests() == ["tests/models/bert", "tests/test_modeling_common.py"]228 229 def test_get_all_tests_on_full_repo(self):230 all_tests = get_all_tests()231 assert "tests/models/albert" in all_tests232 assert "tests/models/bert" in all_tests233 assert "tests/repo_utils" in all_tests234 assert "tests/test_pipeline_mixin.py" in all_tests235 assert "tests/models" not in all_tests236 assert "tests/__pycache__" not in all_tests237 assert "tests/models/albert/test_modeling_albert.py" not in all_tests238 assert "tests/repo_utils/test_tests_fetcher.py" not in all_tests239 240 def test_diff_is_docstring_only(self):241 with tempfile.TemporaryDirectory() as tmp_folder:242 tmp_folder = Path(tmp_folder)243 repo = create_tmp_repo(tmp_folder)244 245 branching_point = repo.refs.main.commit246 bert_file = BERT_MODELING_FILE247 commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING, repo)248 assert diff_is_docstring_only(repo, branching_point, bert_file)249 250 commit_changes(bert_file, BERT_MODEL_FILE_NEW_CODE, repo)251 assert not diff_is_docstring_only(repo, branching_point, bert_file)252 253 def test_get_diff(self):254 with tempfile.TemporaryDirectory() as tmp_folder:255 tmp_folder = Path(tmp_folder)256 repo = create_tmp_repo(tmp_folder)257 258 initial_commit = repo.refs.main.commit259 bert_file = BERT_MODELING_FILE260 commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING, repo)261 assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == []262 263 commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING + "\n# Adding a comment\n", repo)264 assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == []265 266 commit_changes(bert_file, BERT_MODEL_FILE_NEW_CODE, repo)267 assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == [268 "src/transformers/models/bert/modeling_bert.py"269 ]270 271 commit_changes("src/transformers/utils/hub.py", "import huggingface_hub\n\nnew code", repo)272 assert get_diff(repo, repo.head.commit, repo.head.commit.parents) == ["src/transformers/utils/hub.py"]273 assert get_diff(repo, repo.head.commit, [initial_commit]) == [274 "src/transformers/models/bert/modeling_bert.py",275 "src/transformers/utils/hub.py",276 ]277 278 def test_extract_imports_relative(self):279 with tempfile.TemporaryDirectory() as tmp_folder:280 tmp_folder = Path(tmp_folder)281 create_tmp_repo(tmp_folder)282 283 expected_bert_imports = [284 ("src/transformers/modeling_utils.py", ["PreTrainedModel"]),285 ("src/transformers/utils/__init__.py", ["is_torch_available"]),286 ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),287 ]288 expected_utils_imports = [289 ("src/transformers/utils/hub.py", ["cached_file"]),290 ("src/transformers/utils/imports.py", ["is_torch_available"]),291 ]292 with patch_transformer_repo_path(tmp_folder):293 assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports294 assert extract_imports("src/transformers/utils/__init__.py") == expected_utils_imports295 296 with open(tmp_folder / BERT_MODELING_FILE, "w") as f:297 f.write(298 "from ...utils import cached_file, is_torch_available\nfrom .configuration_bert import BertConfig\n"299 )300 expected_bert_imports = [301 ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),302 ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),303 ]304 with patch_transformer_repo_path(tmp_folder):305 assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports306 307 # Test with multi-line imports308 with open(tmp_folder / BERT_MODELING_FILE, "w") as f:309 f.write(310 "from ...utils import (\n cached_file,\n is_torch_available\n)\nfrom .configuration_bert import BertConfig\n"311 )312 expected_bert_imports = [313 ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),314 ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),315 ]316 with patch_transformer_repo_path(tmp_folder):317 assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports318 319 def test_extract_imports_absolute(self):320 with tempfile.TemporaryDirectory() as tmp_folder:321 tmp_folder = Path(tmp_folder)322 create_tmp_repo(tmp_folder)323 324 with open(tmp_folder / BERT_MODELING_FILE, "w") as f:325 f.write(326 "from transformers.utils import cached_file, is_torch_available\nfrom transformers.models.bert.configuration_bert import BertConfig\n"327 )328 expected_bert_imports = [329 ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),330 ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),331 ]332 with patch_transformer_repo_path(tmp_folder):333 assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports334 335 # Test with multi-line imports336 with open(tmp_folder / BERT_MODELING_FILE, "w") as f:337 f.write(338 "from transformers.utils import (\n cached_file,\n is_torch_available\n)\nfrom transformers.models.bert.configuration_bert import BertConfig\n"339 )340 expected_bert_imports = [341 ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),342 ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),343 ]344 with patch_transformer_repo_path(tmp_folder):345 assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports346 347 # Test with base imports348 with open(tmp_folder / BERT_MODELING_FILE, "w") as f:349 f.write(350 "from transformers.utils import (\n cached_file,\n is_torch_available\n)\nfrom transformers import BertConfig\n"351 )352 expected_bert_imports = [353 ("src/transformers/__init__.py", ["BertConfig"]),354 ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),355 ]356 with patch_transformer_repo_path(tmp_folder):357 assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports358 359 def test_get_module_dependencies(self):360 with tempfile.TemporaryDirectory() as tmp_folder:361 tmp_folder = Path(tmp_folder)362 create_tmp_repo(tmp_folder)363 364 expected_bert_dependencies = [365 "src/transformers/modeling_utils.py",366 "src/transformers/models/bert/configuration_bert.py",367 "src/transformers/utils/imports.py",368 ]369 with patch_transformer_repo_path(tmp_folder):370 assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies371 372 expected_test_bert_dependencies = [373 "tests/test_modeling_common.py",374 "src/transformers/models/bert/configuration_bert.py",375 "src/transformers/models/bert/modeling_bert.py",376 ]377 378 with patch_transformer_repo_path(tmp_folder):379 assert (380 get_module_dependencies("tests/models/bert/test_modeling_bert.py")381 == expected_test_bert_dependencies382 )383 384 # Test with a submodule385 (tmp_folder / "src/transformers/utils/logging.py").touch()386 with open(tmp_folder / BERT_MODELING_FILE, "a") as f:387 f.write("from ...utils import logging\n")388 389 expected_bert_dependencies = [390 "src/transformers/modeling_utils.py",391 "src/transformers/models/bert/configuration_bert.py",392 "src/transformers/utils/logging.py",393 "src/transformers/utils/imports.py",394 ]395 with patch_transformer_repo_path(tmp_folder):396 assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies397 398 # Test with an object non-imported in the init399 create_tmp_repo(tmp_folder)400 with open(tmp_folder / BERT_MODELING_FILE, "a") as f:401 f.write("from ...utils import CONSTANT\n")402 403 expected_bert_dependencies = [404 "src/transformers/modeling_utils.py",405 "src/transformers/models/bert/configuration_bert.py",406 "src/transformers/utils/__init__.py",407 "src/transformers/utils/imports.py",408 ]409 with patch_transformer_repo_path(tmp_folder):410 assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies411 412 def test_create_reverse_dependency_tree(self):413 with tempfile.TemporaryDirectory() as tmp_folder:414 tmp_folder = Path(tmp_folder)415 create_tmp_repo(tmp_folder)416 with patch_transformer_repo_path(tmp_folder):417 tree = create_reverse_dependency_tree()418 419 init_edges = [420 "src/transformers/utils/hub.py",421 "src/transformers/utils/imports.py",422 "src/transformers/models/bert/configuration_bert.py",423 "src/transformers/models/bert/modeling_bert.py",424 ]425 assert {f for f, g in tree if g == "src/transformers/__init__.py"} == set(init_edges)426 427 bert_edges = [428 "src/transformers/modeling_utils.py",429 "src/transformers/utils/imports.py",430 "src/transformers/models/bert/configuration_bert.py",431 ]432 assert {f for f, g in tree if g == "src/transformers/models/bert/modeling_bert.py"} == set(bert_edges)433 434 test_bert_edges = [435 "tests/test_modeling_common.py",436 "src/transformers/models/bert/configuration_bert.py",437 "src/transformers/models/bert/modeling_bert.py",438 ]439 assert {f for f, g in tree if g == "tests/models/bert/test_modeling_bert.py"} == set(test_bert_edges)440 441 def test_get_tree_starting_at(self):442 with tempfile.TemporaryDirectory() as tmp_folder:443 tmp_folder = Path(tmp_folder)444 create_tmp_repo(tmp_folder)445 with patch_transformer_repo_path(tmp_folder):446 edges = create_reverse_dependency_tree()447 448 bert_tree = get_tree_starting_at("src/transformers/models/bert/modeling_bert.py", edges)449 config_utils_tree = get_tree_starting_at("src/transformers/configuration_utils.py", edges)450 451 expected_bert_tree = [452 "src/transformers/models/bert/modeling_bert.py",453 [("src/transformers/models/bert/modeling_bert.py", "tests/models/bert/test_modeling_bert.py")],454 ]455 assert bert_tree == expected_bert_tree456 457 expected_config_tree = [458 "src/transformers/configuration_utils.py",459 [("src/transformers/configuration_utils.py", "src/transformers/models/bert/configuration_bert.py")],460 [461 ("src/transformers/models/bert/configuration_bert.py", "tests/models/bert/test_modeling_bert.py"),462 (463 "src/transformers/models/bert/configuration_bert.py",464 "src/transformers/models/bert/modeling_bert.py",465 ),466 ],467 ]468 # Order of the edges is random469 assert [set(v) for v in config_utils_tree] == [set(v) for v in expected_config_tree]470 471 def test_print_tree_deps_of(self):472 with tempfile.TemporaryDirectory() as tmp_folder:473 tmp_folder = Path(tmp_folder)474 create_tmp_repo(tmp_folder)475 476 # There are two possible outputs since the order of the last two lines is non-deterministic.477 expected_std_out = """src/transformers/models/bert/modeling_bert.py478 tests/models/bert/test_modeling_bert.py479src/transformers/configuration_utils.py480 src/transformers/models/bert/configuration_bert.py481 src/transformers/models/bert/modeling_bert.py482 tests/models/bert/test_modeling_bert.py"""483 484 expected_std_out_2 = """src/transformers/models/bert/modeling_bert.py485 tests/models/bert/test_modeling_bert.py486src/transformers/configuration_utils.py487 src/transformers/models/bert/configuration_bert.py488 tests/models/bert/test_modeling_bert.py489 src/transformers/models/bert/modeling_bert.py"""490 491 with patch_transformer_repo_path(tmp_folder), CaptureStdout() as cs:492 print_tree_deps_of("src/transformers/models/bert/modeling_bert.py")493 print_tree_deps_of("src/transformers/configuration_utils.py")494 495 assert cs.out.strip() in [expected_std_out, expected_std_out_2]496 497 def test_create_reverse_dependency_map(self):498 with tempfile.TemporaryDirectory() as tmp_folder:499 tmp_folder = Path(tmp_folder)500 create_tmp_repo(tmp_folder)501 with patch_transformer_repo_path(tmp_folder):502 reverse_map = create_reverse_dependency_map()503 504 # impact of BERT modeling file (note that we stop at the inits and don't go down further)505 expected_bert_deps = {506 "src/transformers/__init__.py",507 "src/transformers/models/bert/__init__.py",508 "tests/models/bert/test_modeling_bert.py",509 }510 assert set(reverse_map["src/transformers/models/bert/modeling_bert.py"]) == expected_bert_deps511 512 # init gets the direct deps (and their recursive deps)513 expected_init_deps = {514 "src/transformers/utils/__init__.py",515 "src/transformers/utils/hub.py",516 "src/transformers/utils/imports.py",517 "src/transformers/models/bert/__init__.py",518 "src/transformers/models/bert/configuration_bert.py",519 "src/transformers/models/bert/modeling_bert.py",520 "src/transformers/configuration_utils.py",521 "src/transformers/modeling_utils.py",522 "tests/test_modeling_common.py",523 "tests/models/bert/test_modeling_bert.py",524 }525 assert set(reverse_map["src/transformers/__init__.py"]) == expected_init_deps526 527 expected_init_deps = {528 "src/transformers/__init__.py",529 "src/transformers/models/bert/configuration_bert.py",530 "src/transformers/models/bert/modeling_bert.py",531 "tests/models/bert/test_modeling_bert.py",532 }533 assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps534 535 # Test that with more models init of bert only gets deps to bert.536 create_tmp_repo(tmp_folder, models=["bert", "gpt2"])537 with patch_transformer_repo_path(tmp_folder):538 reverse_map = create_reverse_dependency_map()539 540 # init gets the direct deps (and their recursive deps)541 expected_init_deps = {542 "src/transformers/__init__.py",543 "src/transformers/models/bert/configuration_bert.py",544 "src/transformers/models/bert/modeling_bert.py",545 "tests/models/bert/test_modeling_bert.py",546 }547 assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps548 549 def test_create_module_to_test_map(self):550 with tempfile.TemporaryDirectory() as tmp_folder:551 tmp_folder = Path(tmp_folder)552 models = models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)]553 create_tmp_repo(tmp_folder, models=models)554 with patch_transformer_repo_path(tmp_folder):555 test_map = create_module_to_test_map(filter_models=True)556 557 for model in models:558 assert test_map[f"src/transformers/models/{model}/modeling_{model}.py"] == [559 f"tests/models/{model}/test_modeling_{model}.py"560 ]561 562 # Init got filtered563 expected_init_tests = {564 "tests/test_modeling_common.py",565 "tests/models/bert/test_modeling_bert.py",566 "tests/models/gpt2/test_modeling_gpt2.py",567 }568 assert set(test_map["src/transformers/__init__.py"]) == expected_init_tests569 570 def test_infer_tests_to_run(self):571 with tempfile.TemporaryDirectory() as tmp_folder:572 tmp_folder = Path(tmp_folder)573 models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)]574 repo = create_tmp_repo(tmp_folder, models=models)575 576 commit_changes("src/transformers/models/bert/modeling_bert.py", BERT_MODEL_FILE_NEW_CODE, repo)577 578 with patch_transformer_repo_path(tmp_folder):579 infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)580 with open(tmp_folder / "test-output.txt", "r") as f:581 tests_to_run = f.read()582 583 assert tests_to_run == "tests/models/bert/test_modeling_bert.py"584 585 # Fake a new model addition586 repo = create_tmp_repo(tmp_folder, models=models)587 588 branch = repo.create_head("new_model")589 branch.checkout()590 591 with open(tmp_folder / "src/transformers/__init__.py", "a") as f:592 f.write("from .models.t5 import T5Config, T5Model\n")593 594 model_dir = tmp_folder / "src/transformers/models/t5"595 model_dir.mkdir(exist_ok=True)596 597 with open(model_dir / "__init__.py", "w") as f:598 f.write("from .configuration_t5 import T5Config\nfrom .modeling_t5 import T5Model\n")599 with open(model_dir / "configuration_t5.py", "w") as f:600 f.write("from ...configuration_utils import PretrainedConfig\ncode")601 with open(model_dir / "modeling_t5.py", "w") as f:602 modeling_code = BERT_MODEL_FILE.replace("bert", "t5").replace("Bert", "T5")603 f.write(modeling_code)604 605 test_dir = tmp_folder / "tests/models/t5"606 test_dir.mkdir(exist_ok=True)607 (test_dir / "__init__.py").touch()608 with open(test_dir / "test_modeling_t5.py", "w") as f:609 f.write(610 "from transformers import T5Config, T5Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode"611 )612 613 repo.index.add(["src", "tests"])614 repo.index.commit("Add T5 model")615 616 with patch_transformer_repo_path(tmp_folder):617 infer_tests_to_run(tmp_folder / "test-output.txt")618 with open(tmp_folder / "test-output.txt", "r") as f:619 tests_to_run = f.read()620 621 expected_tests = {622 "tests/models/bert/test_modeling_bert.py",623 "tests/models/gpt2/test_modeling_gpt2.py",624 "tests/models/t5/test_modeling_t5.py",625 "tests/test_modeling_common.py",626 }627 assert set(tests_to_run.split(" ")) == expected_tests628 629 with patch_transformer_repo_path(tmp_folder):630 infer_tests_to_run(tmp_folder / "test-output.txt", filter_models=False)631 with open(tmp_folder / "test-output.txt", "r") as f:632 tests_to_run = f.read()633 634 expected_tests = [f"tests/models/{name}/test_modeling_{name}.py" for name in models + ["t5"]]635 expected_tests = set(expected_tests + ["tests/test_modeling_common.py"])636 assert set(tests_to_run.split(" ")) == expected_tests637 638 def test_infer_tests_to_run_with_test_modifs(self):639 with tempfile.TemporaryDirectory() as tmp_folder:640 tmp_folder = Path(tmp_folder)641 models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)]642 repo = create_tmp_repo(tmp_folder, models=models)643 644 commit_changes(645 "tests/models/bert/test_modeling_bert.py",646 "from transformers import BertConfig, BertModel\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode1",647 repo,648 )649 650 with patch_transformer_repo_path(tmp_folder):651 infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)652 with open(tmp_folder / "test-output.txt", "r") as f:653 tests_to_run = f.read()654 655 assert tests_to_run == "tests/models/bert/test_modeling_bert.py"656 657 def test_parse_commit_message(self):658 assert parse_commit_message("Normal commit") == {"skip": False, "no_filter": False, "test_all": False}659 660 assert parse_commit_message("[skip ci] commit") == {"skip": True, "no_filter": False, "test_all": False}661 assert parse_commit_message("[ci skip] commit") == {"skip": True, "no_filter": False, "test_all": False}662 assert parse_commit_message("[skip-ci] commit") == {"skip": True, "no_filter": False, "test_all": False}663 assert parse_commit_message("[skip_ci] commit") == {"skip": True, "no_filter": False, "test_all": False}664 665 assert parse_commit_message("[no filter] commit") == {"skip": False, "no_filter": True, "test_all": False}666 assert parse_commit_message("[no-filter] commit") == {"skip": False, "no_filter": True, "test_all": False}667 assert parse_commit_message("[no_filter] commit") == {"skip": False, "no_filter": True, "test_all": False}668 assert parse_commit_message("[filter-no] commit") == {"skip": False, "no_filter": True, "test_all": False}669 670 assert parse_commit_message("[test all] commit") == {"skip": False, "no_filter": False, "test_all": True}671 assert parse_commit_message("[all test] commit") == {"skip": False, "no_filter": False, "test_all": True}672 assert parse_commit_message("[test-all] commit") == {"skip": False, "no_filter": False, "test_all": True}673 assert parse_commit_message("[all_test] commit") == {"skip": False, "no_filter": False, "test_all": True}674