chendl/compositional_test
1
1# coding=utf-82# Copyright 2019 HuggingFace Inc.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import copy17import json18import os19import shutil20import sys21import tempfile22import unittest23import unittest.mock as mock24from pathlib import Path25 26from huggingface_hub import HfFolder, delete_repo27from requests.exceptions import HTTPError28 29from transformers import AutoConfig, BertConfig, GPT2Config, is_torch_available30from transformers.configuration_utils import PretrainedConfig31from transformers.testing_utils import TOKEN, USER, is_staging_test32 33 34sys.path.append(str(Path(__file__).parent.parent / "utils"))35 36from test_module.custom_configuration import CustomConfig # noqa E40237 38 39config_common_kwargs = {40 "return_dict": False,41 "output_hidden_states": True,42 "output_attentions": True,43 "torchscript": True,44 "torch_dtype": "float16",45 "use_bfloat16": True,46 "tf_legacy_loss": True,47 "pruned_heads": {"a": 1},48 "tie_word_embeddings": False,49 "is_decoder": True,50 "cross_attention_hidden_size": 128,51 "add_cross_attention": True,52 "tie_encoder_decoder": True,53 "max_length": 50,54 "min_length": 3,55 "do_sample": True,56 "early_stopping": True,57 "num_beams": 3,58 "num_beam_groups": 3,59 "diversity_penalty": 0.5,60 "temperature": 2.0,61 "top_k": 10,62 "top_p": 0.7,63 "typical_p": 0.2,64 "repetition_penalty": 0.8,65 "length_penalty": 0.8,66 "no_repeat_ngram_size": 5,67 "encoder_no_repeat_ngram_size": 5,68 "bad_words_ids": [1, 2, 3],69 "num_return_sequences": 3,70 "chunk_size_feed_forward": 5,71 "output_scores": True,72 "return_dict_in_generate": True,73 "forced_bos_token_id": 2,74 "forced_eos_token_id": 3,75 "remove_invalid_values": True,76 "architectures": ["BertModel"],77 "finetuning_task": "translation",78 "id2label": {0: "label"},79 "label2id": {"label": "0"},80 "tokenizer_class": "BertTokenizerFast",81 "prefix": "prefix",82 "bos_token_id": 6,83 "pad_token_id": 7,84 "eos_token_id": 8,85 "sep_token_id": 9,86 "decoder_start_token_id": 10,87 "exponential_decay_length_penalty": (5, 1.01),88 "suppress_tokens": [0, 1],89 "begin_suppress_tokens": 2,90 "task_specific_params": {"translation": "some_params"},91 "problem_type": "regression",92}93 94 95class ConfigTester(object):96 def __init__(self, parent, config_class=None, has_text_modality=True, **kwargs):97 self.parent = parent98 self.config_class = config_class99 self.has_text_modality = has_text_modality100 self.inputs_dict = kwargs101 102 def create_and_test_config_common_properties(self):103 config = self.config_class(**self.inputs_dict)104 common_properties = ["hidden_size", "num_attention_heads", "num_hidden_layers"]105 106 # Add common fields for text models107 if self.has_text_modality:108 common_properties.extend(["vocab_size"])109 110 # Test that config has the common properties as getters111 for prop in common_properties:112 self.parent.assertTrue(hasattr(config, prop), msg=f"`{prop}` does not exist")113 114 # Test that config has the common properties as setter115 for idx, name in enumerate(common_properties):116 try:117 setattr(config, name, idx)118 self.parent.assertEqual(119 getattr(config, name), idx, msg=f"`{name} value {idx} expected, but was {getattr(config, name)}"120 )121 except NotImplementedError:122 # Some models might not be able to implement setters for common_properties123 # In that case, a NotImplementedError is raised124 pass125 126 # Test if config class can be called with Config(prop_name=..)127 for idx, name in enumerate(common_properties):128 try:129 config = self.config_class(**{name: idx})130 self.parent.assertEqual(131 getattr(config, name), idx, msg=f"`{name} value {idx} expected, but was {getattr(config, name)}"132 )133 except NotImplementedError:134 # Some models might not be able to implement setters for common_properties135 # In that case, a NotImplementedError is raised136 pass137 138 def create_and_test_config_to_json_string(self):139 config = self.config_class(**self.inputs_dict)140 obj = json.loads(config.to_json_string())141 for key, value in self.inputs_dict.items():142 self.parent.assertEqual(obj[key], value)143 144 def create_and_test_config_to_json_file(self):145 config_first = self.config_class(**self.inputs_dict)146 147 with tempfile.TemporaryDirectory() as tmpdirname:148 json_file_path = os.path.join(tmpdirname, "config.json")149 config_first.to_json_file(json_file_path)150 config_second = self.config_class.from_json_file(json_file_path)151 152 self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())153 154 def create_and_test_config_from_and_save_pretrained(self):155 config_first = self.config_class(**self.inputs_dict)156 157 with tempfile.TemporaryDirectory() as tmpdirname:158 config_first.save_pretrained(tmpdirname)159 config_second = self.config_class.from_pretrained(tmpdirname)160 161 self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())162 163 def create_and_test_config_from_and_save_pretrained_subfolder(self):164 config_first = self.config_class(**self.inputs_dict)165 166 subfolder = "test"167 with tempfile.TemporaryDirectory() as tmpdirname:168 sub_tmpdirname = os.path.join(tmpdirname, subfolder)169 config_first.save_pretrained(sub_tmpdirname)170 config_second = self.config_class.from_pretrained(tmpdirname, subfolder=subfolder)171 172 self.parent.assertEqual(config_second.to_dict(), config_first.to_dict())173 174 def create_and_test_config_with_num_labels(self):175 config = self.config_class(**self.inputs_dict, num_labels=5)176 self.parent.assertEqual(len(config.id2label), 5)177 self.parent.assertEqual(len(config.label2id), 5)178 179 config.num_labels = 3180 self.parent.assertEqual(len(config.id2label), 3)181 self.parent.assertEqual(len(config.label2id), 3)182 183 def check_config_can_be_init_without_params(self):184 if self.config_class.is_composition:185 return186 config = self.config_class()187 self.parent.assertIsNotNone(config)188 189 def check_config_arguments_init(self):190 kwargs = copy.deepcopy(config_common_kwargs)191 config = self.config_class(**kwargs)192 wrong_values = []193 for key, value in config_common_kwargs.items():194 if key == "torch_dtype":195 if not is_torch_available():196 continue197 else:198 import torch199 200 if config.torch_dtype != torch.float16:201 wrong_values.append(("torch_dtype", config.torch_dtype, torch.float16))202 elif getattr(config, key) != value:203 wrong_values.append((key, getattr(config, key), value))204 205 if len(wrong_values) > 0:206 errors = "\n".join([f"- {v[0]}: got {v[1]} instead of {v[2]}" for v in wrong_values])207 raise ValueError(f"The following keys were not properly set in the config:\n{errors}")208 209 def run_common_tests(self):210 self.create_and_test_config_common_properties()211 self.create_and_test_config_to_json_string()212 self.create_and_test_config_to_json_file()213 self.create_and_test_config_from_and_save_pretrained()214 self.create_and_test_config_from_and_save_pretrained_subfolder()215 self.create_and_test_config_with_num_labels()216 self.check_config_can_be_init_without_params()217 self.check_config_arguments_init()218 219 220@is_staging_test221class ConfigPushToHubTester(unittest.TestCase):222 @classmethod223 def setUpClass(cls):224 cls._token = TOKEN225 HfFolder.save_token(TOKEN)226 227 @classmethod228 def tearDownClass(cls):229 try:230 delete_repo(token=cls._token, repo_id="test-config")231 except HTTPError:232 pass233 234 try:235 delete_repo(token=cls._token, repo_id="valid_org/test-config-org")236 except HTTPError:237 pass238 239 try:240 delete_repo(token=cls._token, repo_id="test-dynamic-config")241 except HTTPError:242 pass243 244 def test_push_to_hub(self):245 config = BertConfig(246 vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37247 )248 config.push_to_hub("test-config", use_auth_token=self._token)249 250 new_config = BertConfig.from_pretrained(f"{USER}/test-config")251 for k, v in config.to_dict().items():252 if k != "transformers_version":253 self.assertEqual(v, getattr(new_config, k))254 255 # Reset repo256 delete_repo(token=self._token, repo_id="test-config")257 258 # Push to hub via save_pretrained259 with tempfile.TemporaryDirectory() as tmp_dir:260 config.save_pretrained(tmp_dir, repo_id="test-config", push_to_hub=True, use_auth_token=self._token)261 262 new_config = BertConfig.from_pretrained(f"{USER}/test-config")263 for k, v in config.to_dict().items():264 if k != "transformers_version":265 self.assertEqual(v, getattr(new_config, k))266 267 def test_push_to_hub_in_organization(self):268 config = BertConfig(269 vocab_size=99, hidden_size=32, num_hidden_layers=5, num_attention_heads=4, intermediate_size=37270 )271 config.push_to_hub("valid_org/test-config-org", use_auth_token=self._token)272 273 new_config = BertConfig.from_pretrained("valid_org/test-config-org")274 for k, v in config.to_dict().items():275 if k != "transformers_version":276 self.assertEqual(v, getattr(new_config, k))277 278 # Reset repo279 delete_repo(token=self._token, repo_id="valid_org/test-config-org")280 281 # Push to hub via save_pretrained282 with tempfile.TemporaryDirectory() as tmp_dir:283 config.save_pretrained(284 tmp_dir, repo_id="valid_org/test-config-org", push_to_hub=True, use_auth_token=self._token285 )286 287 new_config = BertConfig.from_pretrained("valid_org/test-config-org")288 for k, v in config.to_dict().items():289 if k != "transformers_version":290 self.assertEqual(v, getattr(new_config, k))291 292 def test_push_to_hub_dynamic_config(self):293 CustomConfig.register_for_auto_class()294 config = CustomConfig(attribute=42)295 296 config.push_to_hub("test-dynamic-config", use_auth_token=self._token)297 298 # This has added the proper auto_map field to the config299 self.assertDictEqual(config.auto_map, {"AutoConfig": "custom_configuration.CustomConfig"})300 301 new_config = AutoConfig.from_pretrained(f"{USER}/test-dynamic-config", trust_remote_code=True)302 # Can't make an isinstance check because the new_config is from the FakeConfig class of a dynamic module303 self.assertEqual(new_config.__class__.__name__, "CustomConfig")304 self.assertEqual(new_config.attribute, 42)305 306 307class ConfigTestUtils(unittest.TestCase):308 def test_config_from_string(self):309 c = GPT2Config()310 311 # attempt to modify each of int/float/bool/str config records and verify they were updated312 n_embd = c.n_embd + 1 # int313 resid_pdrop = c.resid_pdrop + 1.0 # float314 scale_attn_weights = not c.scale_attn_weights # bool315 summary_type = c.summary_type + "foo" # str316 c.update_from_string(317 f"n_embd={n_embd},resid_pdrop={resid_pdrop},scale_attn_weights={scale_attn_weights},summary_type={summary_type}"318 )319 self.assertEqual(n_embd, c.n_embd, "mismatch for key: n_embd")320 self.assertEqual(resid_pdrop, c.resid_pdrop, "mismatch for key: resid_pdrop")321 self.assertEqual(scale_attn_weights, c.scale_attn_weights, "mismatch for key: scale_attn_weights")322 self.assertEqual(summary_type, c.summary_type, "mismatch for key: summary_type")323 324 def test_config_common_kwargs_is_complete(self):325 base_config = PretrainedConfig()326 missing_keys = [key for key in base_config.__dict__ if key not in config_common_kwargs]327 # If this part of the test fails, you have arguments to addin config_common_kwargs above.328 self.assertListEqual(329 missing_keys, ["is_encoder_decoder", "_name_or_path", "_commit_hash", "transformers_version"]330 )331 keys_with_defaults = [key for key, value in config_common_kwargs.items() if value == getattr(base_config, key)]332 if len(keys_with_defaults) > 0:333 raise ValueError(334 "The following keys are set with the default values in"335 " `test_configuration_common.config_common_kwargs` pick another value for them:"336 f" {', '.join(keys_with_defaults)}."337 )338 339 def test_from_pretrained_subfolder(self):340 with self.assertRaises(OSError):341 # config is in subfolder, the following should not work without specifying the subfolder342 _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder")343 344 config = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert-subfolder", subfolder="bert")345 346 self.assertIsNotNone(config)347 348 def test_cached_files_are_used_when_internet_is_down(self):349 # A mock response for an HTTP head request to emulate server down350 response_mock = mock.Mock()351 response_mock.status_code = 500352 response_mock.headers = {}353 response_mock.raise_for_status.side_effect = HTTPError354 response_mock.json.return_value = {}355 356 # Download this model to make sure it's in the cache.357 _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")358 359 # Under the mock environment we get a 500 error when trying to reach the model.360 with mock.patch("requests.request", return_value=response_mock) as mock_head:361 _ = BertConfig.from_pretrained("hf-internal-testing/tiny-random-bert")362 # This check we did call the fake head request363 mock_head.assert_called()364 365 def test_legacy_load_from_url(self):366 # This test is for deprecated behavior and can be removed in v5367 _ = BertConfig.from_pretrained(368 "https://huggingface.co/hf-internal-testing/tiny-random-bert/resolve/main/config.json"369 )370 371 372class ConfigurationVersioningTest(unittest.TestCase):373 def test_local_versioning(self):374 configuration = AutoConfig.from_pretrained("bert-base-cased")375 configuration.configuration_files = ["config.4.0.0.json"]376 377 with tempfile.TemporaryDirectory() as tmp_dir:378 configuration.save_pretrained(tmp_dir)379 configuration.hidden_size = 2380 json.dump(configuration.to_dict(), open(os.path.join(tmp_dir, "config.4.0.0.json"), "w"))381 382 # This should pick the new configuration file as the version of Transformers is > 4.0.0383 new_configuration = AutoConfig.from_pretrained(tmp_dir)384 self.assertEqual(new_configuration.hidden_size, 2)385 386 # Will need to be adjusted if we reach v42 and this test is still here.387 # Should pick the old configuration file as the version of Transformers is < 4.42.0388 configuration.configuration_files = ["config.42.0.0.json"]389 configuration.hidden_size = 768390 configuration.save_pretrained(tmp_dir)391 shutil.move(os.path.join(tmp_dir, "config.4.0.0.json"), os.path.join(tmp_dir, "config.42.0.0.json"))392 new_configuration = AutoConfig.from_pretrained(tmp_dir)393 self.assertEqual(new_configuration.hidden_size, 768)394 395 def test_repo_versioning_before(self):396 # This repo has two configuration files, one for v4.0.0 and above with a different hidden size.397 repo = "hf-internal-testing/test-two-configs"398 399 import transformers as new_transformers400 401 new_transformers.configuration_utils.__version__ = "v4.0.0"402 new_configuration, kwargs = new_transformers.models.auto.AutoConfig.from_pretrained(403 repo, return_unused_kwargs=True404 )405 self.assertEqual(new_configuration.hidden_size, 2)406 # This checks `_configuration_file` ia not kept in the kwargs by mistake.407 self.assertDictEqual(kwargs, {})408 409 # Testing an older version by monkey-patching the version in the module it's used.410 import transformers as old_transformers411 412 old_transformers.configuration_utils.__version__ = "v3.0.0"413 old_configuration = old_transformers.models.auto.AutoConfig.from_pretrained(repo)414 self.assertEqual(old_configuration.hidden_size, 768)415