chendl/compositional_test
1
1# coding=utf-82# Copyright 2021 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16"""17Welcome to tests_fetcher V2.18This util is designed to fetch tests to run on a PR so that only the tests impacted by the modifications are run, and19when too many models are being impacted, only run the tests of a subset of core models. It works like this.20 21Stage 1: Identify the modified files. This takes all the files from the branching point to the current commit (so22all modifications in a PR, not just the last commit) but excludes modifications that are on docstrings or comments23only.24 25Stage 2: Extract the tests to run. This is done by looking at the imports in each module and test file: if module A26imports module B, then changing module B impacts module A, so the tests using module A should be run. We thus get the27dependencies of each model and then recursively builds the 'reverse' map of dependencies to get all modules and tests28impacted by a given file. We then only keep the tests (and only the code models tests if there are too many modules).29 30Caveats:31 - This module only filters tests by files (not individual tests) so it's better to have tests for different things32 in different files.33 - This module assumes inits are just importing things, not really building objects, so it's better to structure34 them this way and move objects building in separate submodules.35"""36 37import argparse38import collections39import json40import os41import re42from contextlib import contextmanager43from pathlib import Path44 45from git import Repo46 47 48PATH_TO_REPO = Path(__file__).parent.parent.resolve()49PATH_TO_TRANFORMERS = PATH_TO_REPO / "src/transformers"50PATH_TO_TESTS = PATH_TO_REPO / "tests"51 52# List here the models to always test.53IMPORTANT_MODELS = [54 # Most downloaded models55 "bert",56 "clip",57 "t5",58 "xlm-roberta",59 "gpt2",60 "bart",61 "mpnet",62 "gpt-j",63 "wav2vec2",64 "deberta-v2",65 "layoutlm",66 "opt",67 "longformer",68 "vit",69 # Pipeline-specific model (to be sure each pipeline has one model in this list)70 "tapas",71 "vilt",72 "clap",73 "detr",74 "owlvit",75 "dpt",76 "videomae",77]78 79 80@contextmanager81def checkout_commit(repo, commit_id):82 """83 Context manager that checks out a commit in the repo.84 """85 current_head = repo.head.commit if repo.head.is_detached else repo.head.ref86 87 try:88 repo.git.checkout(commit_id)89 yield90 91 finally:92 repo.git.checkout(current_head)93 94 95def clean_code(content):96 """97 Remove docstrings, empty line or comments from `content`.98 """99 # fmt: off100 # Remove docstrings by splitting on triple " then triple ':101 splits = content.split('\"\"\"')102 content = "".join(splits[::2])103 splits = content.split("\'\'\'")104 # fmt: on105 content = "".join(splits[::2])106 107 # Remove empty lines and comments108 lines_to_keep = []109 for line in content.split("\n"):110 # remove anything that is after a # sign.111 line = re.sub("#.*$", "", line)112 if len(line) == 0 or line.isspace():113 continue114 lines_to_keep.append(line)115 return "\n".join(lines_to_keep)116 117 118def get_all_tests():119 """120 Return a list of paths to all test folders and files under `tests`. All paths are rooted at `tests`.121 122 - folders under `tests`: `tokenization`, `pipelines`, etc. The folder `models` is excluded.123 - folders under `tests/models`: `bert`, `gpt2`, etc.124 - test files under `tests`: `test_modeling_common.py`, `test_tokenization_common.py`, etc.125 """126 127 # test folders/files directly under `tests` folder128 tests = os.listdir(PATH_TO_TESTS)129 tests = [f"tests/{f}" for f in tests if "__pycache__" not in f]130 tests = sorted([f for f in tests if (PATH_TO_REPO / f).is_dir() or f.startswith("tests/test_")])131 132 # model specific test folders133 model_test_folders = os.listdir(PATH_TO_TESTS / "models")134 model_test_folders = [f"tests/models/{f}" for f in model_test_folders if "__pycache__" not in f]135 model_test_folders = sorted([f for f in model_test_folders if (PATH_TO_REPO / f).is_dir()])136 137 tests.remove("tests/models")138 # Sagemaker tests are not meant to be run on the CI.139 if "tests/sagemaker" in tests:140 tests.remove("tests/sagemaker")141 tests = model_test_folders + tests142 143 return tests144 145 146def diff_is_docstring_only(repo, branching_point, filename):147 """148 Check if the diff is only in docstrings in a filename.149 """150 folder = Path(repo.working_dir)151 with checkout_commit(repo, branching_point):152 with open(folder / filename, "r", encoding="utf-8") as f:153 old_content = f.read()154 155 with open(folder / filename, "r", encoding="utf-8") as f:156 new_content = f.read()157 158 old_content_clean = clean_code(old_content)159 new_content_clean = clean_code(new_content)160 161 return old_content_clean == new_content_clean162 163 164def get_diff(repo, base_commit, commits):165 """166 Get's the diff between one or several commits and the head of the repository.167 """168 print("\n### DIFF ###\n")169 code_diff = []170 for commit in commits:171 for diff_obj in commit.diff(base_commit):172 # We always add new python files173 if diff_obj.change_type == "A" and diff_obj.b_path.endswith(".py"):174 code_diff.append(diff_obj.b_path)175 # We check that deleted python files won't break corresponding tests.176 elif diff_obj.change_type == "D" and diff_obj.a_path.endswith(".py"):177 code_diff.append(diff_obj.a_path)178 # Now for modified files179 elif diff_obj.change_type in ["M", "R"] and diff_obj.b_path.endswith(".py"):180 # In case of renames, we'll look at the tests using both the old and new name.181 if diff_obj.a_path != diff_obj.b_path:182 code_diff.extend([diff_obj.a_path, diff_obj.b_path])183 else:184 # Otherwise, we check modifications are in code and not docstrings.185 if diff_is_docstring_only(repo, commit, diff_obj.b_path):186 print(f"Ignoring diff in {diff_obj.b_path} as it only concerns docstrings or comments.")187 else:188 code_diff.append(diff_obj.a_path)189 190 return code_diff191 192 193def get_modified_python_files(diff_with_last_commit=False):194 """195 Return a list of python files that have been modified between:196 197 - the current head and the main branch if `diff_with_last_commit=False` (default)198 - the current head and its parent commit otherwise.199 """200 repo = Repo(PATH_TO_REPO)201 202 if not diff_with_last_commit:203 print(f"main is at {repo.refs.main.commit}")204 print(f"Current head is at {repo.head.commit}")205 206 branching_commits = repo.merge_base(repo.refs.main, repo.head)207 for commit in branching_commits:208 print(f"Branching commit: {commit}")209 return get_diff(repo, repo.head.commit, branching_commits)210 else:211 print(f"main is at {repo.head.commit}")212 parent_commits = repo.head.commit.parents213 for commit in parent_commits:214 print(f"Parent commit: {commit}")215 return get_diff(repo, repo.head.commit, parent_commits)216 217 218# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line.219# \s*from\s+(\.+\S+)\s+import\s+([^\n]+) -> Line only contains from .xxx import yyy and we catch .xxx and yyy220# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every221# other import.222_re_single_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+([^\n]+)(?=\n)")223# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line.224# \s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\) -> Line continues with from .xxx import (yyy) and we catch .xxx and yyy225# yyy will take multiple lines otherwise there wouldn't be parenthesis.226_re_multi_line_relative_imports = re.compile(r"(?:^|\n)\s*from\s+(\.+\S+)\s+import\s+\(([^\)]+)\)")227# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line.228# \s*from\s+transformers(\S*)\s+import\s+([^\n]+) -> Line only contains from transformers.xxx import yyy and we catch229# .xxx and yyy230# (?=\n) -> Look-ahead to a new line. We can't just put \n here or using find_all on this re will only catch every231# other import.232_re_single_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+transformers(\S*)\s+import\s+([^\n]+)(?=\n)")233# (:?^|\n) -> Non-catching group for the beginning of the doc or a new line.234# \s*from\s+transformers(\S*)\s+import\s+\(([^\)]+)\) -> Line continues with from transformers.xxx import (yyy) and we235# catch .xxx and yyy. yyy will take multiple lines otherwise there wouldn't be parenthesis.236_re_multi_line_direct_imports = re.compile(r"(?:^|\n)\s*from\s+transformers(\S*)\s+import\s+\(([^\)]+)\)")237 238 239def extract_imports(module_fname, cache=None):240 """241 Get the imports a given module makes. This takes a module filename and returns the list of module filenames242 imported in the module with the objects imported in that module filename.243 """244 if cache is not None and module_fname in cache:245 return cache[module_fname]246 247 with open(PATH_TO_REPO / module_fname, "r", encoding="utf-8") as f:248 content = f.read()249 250 # Filter out all docstrings to not get imports in code examples.251 splits = content.split('"""')252 content = "".join(splits[::2])253 254 module_parts = str(module_fname).split(os.path.sep)255 imported_modules = []256 257 # Let's start with relative imports258 relative_imports = _re_single_line_relative_imports.findall(content)259 relative_imports = [260 (mod, imp) for mod, imp in relative_imports if "# tests_ignore" not in imp and imp.strip() != "("261 ]262 multiline_relative_imports = _re_multi_line_relative_imports.findall(content)263 relative_imports += [(mod, imp) for mod, imp in multiline_relative_imports if "# tests_ignore" not in imp]264 265 for module, imports in relative_imports:266 level = 0267 while module.startswith("."):268 module = module[1:]269 level += 1270 271 if len(module) > 0:272 dep_parts = module_parts[: len(module_parts) - level] + module.split(".")273 else:274 dep_parts = module_parts[: len(module_parts) - level]275 imported_module = os.path.sep.join(dep_parts)276 imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")]))277 278 # Let's continue with direct imports279 direct_imports = _re_single_line_direct_imports.findall(content)280 direct_imports = [(mod, imp) for mod, imp in direct_imports if "# tests_ignore" not in imp and imp.strip() != "("]281 multiline_direct_imports = _re_multi_line_direct_imports.findall(content)282 direct_imports += [(mod, imp) for mod, imp in multiline_direct_imports if "# tests_ignore" not in imp]283 284 for module, imports in direct_imports:285 import_parts = module.split(".")[1:] # ignore the first .286 dep_parts = ["src", "transformers"] + import_parts287 imported_module = os.path.sep.join(dep_parts)288 imported_modules.append((imported_module, [imp.strip() for imp in imports.split(",")]))289 290 result = []291 for module_file, imports in imported_modules:292 if (PATH_TO_REPO / f"{module_file}.py").is_file():293 module_file = f"{module_file}.py"294 elif (PATH_TO_REPO / module_file).is_dir() and (PATH_TO_REPO / module_file / "__init__.py").is_file():295 module_file = os.path.sep.join([module_file, "__init__.py"])296 imports = [imp for imp in imports if len(imp) > 0 and re.match("^[A-Za-z0-9_]*$", imp)]297 if len(imports) > 0:298 result.append((module_file, imports))299 300 if cache is not None:301 cache[module_fname] = result302 303 return result304 305 306def get_module_dependencies(module_fname, cache=None):307 """308 Get the dependencies of a module from the module filename as a list of module filenames. This will resolve any309 __init__ we pass: if we import from a submodule utils, the dependencies will be utils/foo.py and utils/bar.py (if310 the objects imported actually come from utils.foo and utils.bar) not utils/__init__.py.311 """312 dependencies = []313 imported_modules = extract_imports(module_fname, cache=cache)314 # The while loop is to recursively traverse all inits we may encounter.315 while len(imported_modules) > 0:316 new_modules = []317 for module, imports in imported_modules:318 # If we end up in an __init__ we are often not actually importing from this init (except in the case where319 # the object is fully defined in the __init__)320 if module.endswith("__init__.py"):321 # So we get the imports from that init then try to find where our objects come from.322 new_imported_modules = extract_imports(module, cache=cache)323 for new_module, new_imports in new_imported_modules:324 if any([i in new_imports for i in imports]):325 if new_module not in dependencies:326 new_modules.append((new_module, [i for i in new_imports if i in imports]))327 imports = [i for i in imports if i not in new_imports]328 if len(imports) > 0:329 # If there are any objects lefts, they may be a submodule330 path_to_module = PATH_TO_REPO / module.replace("__init__.py", "")331 dependencies.extend(332 [333 os.path.join(module.replace("__init__.py", ""), f"{i}.py")334 for i in imports335 if (path_to_module / f"{i}.py").is_file()336 ]337 )338 imports = [i for i in imports if not (path_to_module / f"{i}.py").is_file()]339 if len(imports) > 0:340 # Then if there are still objects left, they are fully defined in the init, so we keep it as a341 # dependency.342 dependencies.append(module)343 else:344 dependencies.append(module)345 346 imported_modules = new_modules347 return dependencies348 349 350def create_reverse_dependency_tree():351 """352 Create a list of all edges (a, b) which mean that modifying a impacts b with a going over all module and test files.353 """354 cache = {}355 all_modules = list(PATH_TO_TRANFORMERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py"))356 all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules]357 edges = [(dep, mod) for mod in all_modules for dep in get_module_dependencies(mod, cache=cache)]358 359 return list(set(edges))360 361 362def get_tree_starting_at(module, edges):363 """364 Returns the tree starting at a given module following all edges in the following format: [module, [list of edges365 starting at module], [list of edges starting at the preceding level], ...]366 """367 vertices_seen = [module]368 new_edges = [edge for edge in edges if edge[0] == module and edge[1] != module and "__init__.py" not in edge[1]]369 tree = [module]370 while len(new_edges) > 0:371 tree.append(new_edges)372 final_vertices = list({edge[1] for edge in new_edges})373 vertices_seen.extend(final_vertices)374 new_edges = [375 edge376 for edge in edges377 if edge[0] in final_vertices and edge[1] not in vertices_seen and "__init__.py" not in edge[1]378 ]379 380 return tree381 382 383def print_tree_deps_of(module, all_edges=None):384 """385 Prints the tree of modules depending on a given module.386 """387 if all_edges is None:388 all_edges = create_reverse_dependency_tree()389 tree = get_tree_starting_at(module, all_edges)390 391 # The list of lines is a list of tuples (line_to_be_printed, module)392 # Keeping the modules lets us know where to insert each new lines in the list.393 lines = [(tree[0], tree[0])]394 for index in range(1, len(tree)):395 edges = tree[index]396 start_edges = {edge[0] for edge in edges}397 398 for start in start_edges:399 end_edges = {edge[1] for edge in edges if edge[0] == start}400 # We will insert all those edges just after the line showing start.401 pos = 0402 while lines[pos][1] != start:403 pos += 1404 lines = lines[: pos + 1] + [(" " * (2 * index) + end, end) for end in end_edges] + lines[pos + 1 :]405 406 for line in lines:407 # We don't print the refs that where just here to help build lines.408 print(line[0])409 410 411def create_reverse_dependency_map():412 """413 Create the dependency map from module/test filename to the list of modules/tests that depend on it (even414 recursively).415 """416 cache = {}417 all_modules = list(PATH_TO_TRANFORMERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py"))418 all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules]419 direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules}420 421 # This recurses the dependencies422 something_changed = True423 while something_changed:424 something_changed = False425 for m in all_modules:426 for d in direct_deps[m]:427 if d.endswith("__init__.py"):428 continue429 if d not in direct_deps:430 raise ValueError(f"KeyError:{d}. From {m}")431 new_deps = set(direct_deps[d]) - set(direct_deps[m])432 if len(new_deps) > 0:433 direct_deps[m].extend(list(new_deps))434 something_changed = True435 436 # Finally we can build the reverse map.437 reverse_map = collections.defaultdict(list)438 for m in all_modules:439 for d in direct_deps[m]:440 reverse_map[d].append(m)441 442 for m in [f for f in all_modules if f.endswith("__init__.py")]:443 direct_deps = get_module_dependencies(m, cache=cache)444 deps = sum([reverse_map[d] for d in direct_deps if not d.endswith("__init__.py")], direct_deps)445 reverse_map[m] = list(set(deps) - {m})446 447 return reverse_map448 449 450def create_module_to_test_map(reverse_map=None, filter_models=False):451 """452 Extract the tests from the reverse_dependency_map and potentially filters the model tests.453 """454 if reverse_map is None:455 reverse_map = create_reverse_dependency_map()456 test_map = {module: [f for f in deps if f.startswith("tests")] for module, deps in reverse_map.items()}457 458 if not filter_models:459 return test_map460 461 num_model_tests = len(list(PATH_TO_TESTS.glob("models/*")))462 463 def has_many_models(tests):464 model_tests = {Path(t).parts[2] for t in tests if t.startswith("tests/models/")}465 return len(model_tests) > num_model_tests // 2466 467 def filter_tests(tests):468 return [t for t in tests if not t.startswith("tests/models/") or Path(t).parts[2] in IMPORTANT_MODELS]469 470 return {module: (filter_tests(tests) if has_many_models(tests) else tests) for module, tests in test_map.items()}471 472 473def check_imports_all_exist():474 """475 Isn't used per se by the test fetcher but might be used later as a quality check. Putting this here for now so the476 code is not lost.477 """478 cache = {}479 all_modules = list(PATH_TO_TRANFORMERS.glob("**/*.py")) + list(PATH_TO_TESTS.glob("**/*.py"))480 all_modules = [str(mod.relative_to(PATH_TO_REPO)) for mod in all_modules]481 direct_deps = {m: get_module_dependencies(m, cache=cache) for m in all_modules}482 483 for module, deps in direct_deps.items():484 for dep in deps:485 if not (PATH_TO_REPO / dep).is_file():486 print(f"{module} has dependency on {dep} which does not exist.")487 488 489def _print_list(l):490 return "\n".join([f"- {f}" for f in l])491 492 493def create_json_map(test_files_to_run, json_output_file):494 if json_output_file is None:495 return496 497 test_map = {}498 for test_file in test_files_to_run:499 # `test_file` is a path to a test folder/file, starting with `tests/`. For example,500 # - `tests/models/bert/test_modeling_bert.py` or `tests/models/bert`501 # - `tests/trainer/test_trainer.py` or `tests/trainer`502 # - `tests/test_modeling_common.py`503 names = test_file.split(os.path.sep)504 if names[1] == "models":505 # take the part like `models/bert` for modeling tests506 key = os.path.sep.join(names[1:3])507 elif len(names) > 2 or not test_file.endswith(".py"):508 # test folders under `tests` or python files under them509 # take the part like tokenization, `pipeline`, etc. for other test categories510 key = os.path.sep.join(names[1:2])511 else:512 # common test files directly under `tests/`513 key = "common"514 515 if key not in test_map:516 test_map[key] = []517 test_map[key].append(test_file)518 519 # sort the keys & values520 keys = sorted(test_map.keys())521 test_map = {k: " ".join(sorted(test_map[k])) for k in keys}522 with open(json_output_file, "w", encoding="UTF-8") as fp:523 json.dump(test_map, fp, ensure_ascii=False)524 525 526def infer_tests_to_run(527 output_file, diff_with_last_commit=False, filters=None, filter_models=True, json_output_file=None528):529 modified_files = get_modified_python_files(diff_with_last_commit=diff_with_last_commit)530 print(f"\n### MODIFIED FILES ###\n{_print_list(modified_files)}")531 532 # Create the map that will give us all impacted modules.533 reverse_map = create_reverse_dependency_map()534 impacted_files = modified_files.copy()535 for f in modified_files:536 if f in reverse_map:537 impacted_files.extend(reverse_map[f])538 539 # Remove duplicates540 impacted_files = sorted(set(impacted_files))541 print(f"\n### IMPACTED FILES ###\n{_print_list(impacted_files)}")542 543 # Grab the corresponding test files:544 if "setup.py" in modified_files:545 test_files_to_run = ["tests"]546 repo_utils_launch = True547 else:548 # All modified tests need to be run.549 test_files_to_run = [550 f for f in modified_files if f.startswith("tests") and f.split(os.path.sep)[-1].startswith("test")551 ]552 # Then we grab the corresponding test files.553 test_map = create_module_to_test_map(reverse_map=reverse_map, filter_models=filter_models)554 for f in modified_files:555 if f in test_map:556 test_files_to_run.extend(test_map[f])557 test_files_to_run = sorted(set(test_files_to_run))558 # Remove SageMaker tests559 test_files_to_run = [f for f in test_files_to_run if not f.split(os.path.sep)[1] == "sagemaker"]560 # Make sure we did not end up with a test file that was removed561 test_files_to_run = [f for f in test_files_to_run if (PATH_TO_REPO / f).exists()]562 if filters is not None:563 filtered_files = []564 for _filter in filters:565 filtered_files.extend([f for f in test_files_to_run if f.startswith(_filter)])566 test_files_to_run = filtered_files567 568 repo_utils_launch = any(f.split(os.path.sep)[1] == "repo_utils" for f in modified_files)569 570 if repo_utils_launch:571 repo_util_file = Path(output_file).parent / "test_repo_utils.txt"572 with open(repo_util_file, "w", encoding="utf-8") as f:573 f.write("tests/repo_utils")574 575 print(f"\n### TEST TO RUN ###\n{_print_list(test_files_to_run)}")576 if len(test_files_to_run) > 0:577 with open(output_file, "w", encoding="utf-8") as f:578 f.write(" ".join(test_files_to_run))579 580 # Create a map that maps test categories to test files, i.e. `models/bert` -> [...test_modeling_bert.py, ...]581 582 # Get all test directories (and some common test files) under `tests` and `tests/models` if `test_files_to_run`583 # contains `tests` (i.e. when `setup.py` is changed).584 if "tests" in test_files_to_run:585 test_files_to_run = get_all_tests()586 587 create_json_map(test_files_to_run, json_output_file)588 589 590def filter_tests(output_file, filters):591 """592 Reads the content of the output file and filters out all the tests in a list of given folders.593 594 Args:595 output_file (`str` or `os.PathLike`): The path to the output file of the tests fetcher.596 filters (`List[str]`): A list of folders to filter.597 """598 if not os.path.isfile(output_file):599 print("No test file found.")600 return601 with open(output_file, "r", encoding="utf-8") as f:602 test_files = f.read().split(" ")603 604 if len(test_files) == 0 or test_files == [""]:605 print("No tests to filter.")606 return607 608 if test_files == ["tests"]:609 test_files = [os.path.join("tests", f) for f in os.listdir("tests") if f not in ["__init__.py"] + filters]610 else:611 test_files = [f for f in test_files if f.split(os.path.sep)[1] not in filters]612 613 with open(output_file, "w", encoding="utf-8") as f:614 f.write(" ".join(test_files))615 616 617def parse_commit_message(commit_message):618 """619 Parses the commit message to detect if a command is there to skip, force all or part of the CI.620 621 Returns a dictionary of strings to bools with keys skip, test_all_models and test_all.622 """623 if commit_message is None:624 return {"skip": False, "no_filter": False, "test_all": False}625 626 command_search = re.search(r"\[([^\]]*)\]", commit_message)627 if command_search is not None:628 command = command_search.groups()[0]629 command = command.lower().replace("-", " ").replace("_", " ")630 skip = command in ["ci skip", "skip ci", "circleci skip", "skip circleci"]631 no_filter = set(command.split(" ")) == {"no", "filter"}632 test_all = set(command.split(" ")) == {"test", "all"}633 return {"skip": skip, "no_filter": no_filter, "test_all": test_all}634 else:635 return {"skip": False, "no_filter": False, "test_all": False}636 637 638if __name__ == "__main__":639 parser = argparse.ArgumentParser()640 parser.add_argument(641 "--output_file", type=str, default="test_list.txt", help="Where to store the list of tests to run"642 )643 parser.add_argument(644 "--json_output_file",645 type=str,646 default="test_map.json",647 help="Where to store the tests to run in a dictionary format mapping test categories to test files",648 )649 parser.add_argument(650 "--diff_with_last_commit",651 action="store_true",652 help="To fetch the tests between the current commit and the last commit",653 )654 parser.add_argument(655 "--filters",656 type=str,657 nargs="*",658 default=["tests"],659 help="Only keep the test files matching one of those filters.",660 )661 parser.add_argument(662 "--filter_tests",663 action="store_true",664 help="Will filter the pipeline/repo utils tests outside of the generated list of tests.",665 )666 parser.add_argument(667 "--print_dependencies_of",668 type=str,669 help="Will only print the tree of modules depending on the file passed.",670 default=None,671 )672 parser.add_argument(673 "--commit_message",674 type=str,675 help="The commit message (which could contain a command to force all tests or skip the CI).",676 default=None,677 )678 args = parser.parse_args()679 if args.print_dependencies_of is not None:680 print_tree_deps_of(args.print_dependencies_of)681 elif args.filter_tests:682 filter_tests(args.output_file, ["pipelines", "repo_utils"])683 else:684 repo = Repo(PATH_TO_REPO)685 commit_message = repo.head.commit.message686 commit_flags = parse_commit_message(commit_message)687 if commit_flags["skip"]:688 print("Force-skipping the CI")689 quit()690 if commit_flags["no_filter"]:691 print("Running all tests fetched without filtering.")692 if commit_flags["test_all"]:693 print("Force-launching all tests")694 695 diff_with_last_commit = args.diff_with_last_commit696 if not diff_with_last_commit and not repo.head.is_detached and repo.head.ref == repo.refs.main:697 print("main branch detected, fetching tests against last commit.")698 diff_with_last_commit = True699 700 if not commit_flags["test_all"]:701 try:702 infer_tests_to_run(703 args.output_file,704 diff_with_last_commit=diff_with_last_commit,705 filters=args.filters,706 json_output_file=args.json_output_file,707 filter_models=not commit_flags["no_filter"],708 )709 filter_tests(args.output_file, ["repo_utils"])710 except Exception as e:711 print(f"\nError when trying to grab the relevant tests: {e}\n\nRunning all tests.")712 commit_flags["test_all"] = True713 714 if commit_flags["test_all"]:715 with open(args.output_file, "w", encoding="utf-8") as f:716 if args.filters is None:717 f.write("./tests/")718 else:719 f.write(" ".join(args.filters))720 721 test_files_to_run = get_all_tests()722 create_json_map(test_files_to_run, args.json_output_file)723