declare-lab/tango2
92
1# coding=utf-82# Copyright 2023 The HuggingFace Inc. team.3#4# Licensed under the Apache License, Version 2.0 (the "License");5# you may not use this file except in compliance with the License.6# You may obtain a copy of the License at7#8# http://www.apache.org/licenses/LICENSE-2.09#10# Unless required by applicable law or agreed to in writing, software11# distributed under the License is distributed on an "AS IS" BASIS,12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.13# See the License for the specific language governing permissions and14# limitations under the License.15 16import os17from pickle import UnpicklingError18from typing import Any, Dict, Union19 20import jax21import jax.numpy as jnp22import msgpack.exceptions23from flax.core.frozen_dict import FrozenDict, unfreeze24from flax.serialization import from_bytes, to_bytes25from flax.traverse_util import flatten_dict, unflatten_dict26from huggingface_hub import hf_hub_download27from huggingface_hub.utils import EntryNotFoundError, RepositoryNotFoundError, RevisionNotFoundError28from requests import HTTPError29 30from .. import __version__, is_torch_available31from ..utils import (32 CONFIG_NAME,33 DIFFUSERS_CACHE,34 FLAX_WEIGHTS_NAME,35 HUGGINGFACE_CO_RESOLVE_ENDPOINT,36 WEIGHTS_NAME,37 logging,38)39from .modeling_flax_pytorch_utils import convert_pytorch_state_dict_to_flax40 41 42logger = logging.get_logger(__name__)43 44 45class FlaxModelMixin:46 r"""47 Base class for all flax models.48 49 [`FlaxModelMixin`] takes care of storing the configuration of the models and handles methods for loading,50 downloading and saving models.51 """52 config_name = CONFIG_NAME53 _automatically_saved_args = ["_diffusers_version", "_class_name", "_name_or_path"]54 _flax_internal_args = ["name", "parent", "dtype"]55 56 @classmethod57 def _from_config(cls, config, **kwargs):58 """59 All context managers that the model should be initialized under go here.60 """61 return cls(config, **kwargs)62 63 def _cast_floating_to(self, params: Union[Dict, FrozenDict], dtype: jnp.dtype, mask: Any = None) -> Any:64 """65 Helper method to cast floating-point values of given parameter `PyTree` to given `dtype`.66 """67 68 # taken from https://github.com/deepmind/jmp/blob/3a8318abc3292be38582794dbf7b094e6583b192/jmp/_src/policy.py#L2769 def conditional_cast(param):70 if isinstance(param, jnp.ndarray) and jnp.issubdtype(param.dtype, jnp.floating):71 param = param.astype(dtype)72 return param73 74 if mask is None:75 return jax.tree_map(conditional_cast, params)76 77 flat_params = flatten_dict(params)78 flat_mask, _ = jax.tree_flatten(mask)79 80 for masked, key in zip(flat_mask, flat_params.keys()):81 if masked:82 param = flat_params[key]83 flat_params[key] = conditional_cast(param)84 85 return unflatten_dict(flat_params)86 87 def to_bf16(self, params: Union[Dict, FrozenDict], mask: Any = None):88 r"""89 Cast the floating-point `params` to `jax.numpy.bfloat16`. This returns a new `params` tree and does not cast90 the `params` in place.91 92 This method can be used on TPU to explicitly convert the model parameters to bfloat16 precision to do full93 half-precision training or to save weights in bfloat16 for inference in order to save memory and improve speed.94 95 Arguments:96 params (`Union[Dict, FrozenDict]`):97 A `PyTree` of model parameters.98 mask (`Union[Dict, FrozenDict]`):99 A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for params100 you want to cast, and should be `False` for those you want to skip.101 102 Examples:103 104 ```python105 >>> from diffusers import FlaxUNet2DConditionModel106 107 >>> # load model108 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5")109 >>> # By default, the model parameters will be in fp32 precision, to cast these to bfloat16 precision110 >>> params = model.to_bf16(params)111 >>> # If you don't want to cast certain parameters (for example layer norm bias and scale)112 >>> # then pass the mask as follows113 >>> from flax import traverse_util114 115 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5")116 >>> flat_params = traverse_util.flatten_dict(params)117 >>> mask = {118 ... path: (path[-2] != ("LayerNorm", "bias") and path[-2:] != ("LayerNorm", "scale"))119 ... for path in flat_params120 ... }121 >>> mask = traverse_util.unflatten_dict(mask)122 >>> params = model.to_bf16(params, mask)123 ```"""124 return self._cast_floating_to(params, jnp.bfloat16, mask)125 126 def to_fp32(self, params: Union[Dict, FrozenDict], mask: Any = None):127 r"""128 Cast the floating-point `params` to `jax.numpy.float32`. This method can be used to explicitly convert the129 model parameters to fp32 precision. This returns a new `params` tree and does not cast the `params` in place.130 131 Arguments:132 params (`Union[Dict, FrozenDict]`):133 A `PyTree` of model parameters.134 mask (`Union[Dict, FrozenDict]`):135 A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for params136 you want to cast, and should be `False` for those you want to skip137 138 Examples:139 140 ```python141 >>> from diffusers import FlaxUNet2DConditionModel142 143 >>> # Download model and configuration from huggingface.co144 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5")145 >>> # By default, the model params will be in fp32, to illustrate the use of this method,146 >>> # we'll first cast to fp16 and back to fp32147 >>> params = model.to_f16(params)148 >>> # now cast back to fp32149 >>> params = model.to_fp32(params)150 ```"""151 return self._cast_floating_to(params, jnp.float32, mask)152 153 def to_fp16(self, params: Union[Dict, FrozenDict], mask: Any = None):154 r"""155 Cast the floating-point `params` to `jax.numpy.float16`. This returns a new `params` tree and does not cast the156 `params` in place.157 158 This method can be used on GPU to explicitly convert the model parameters to float16 precision to do full159 half-precision training or to save weights in float16 for inference in order to save memory and improve speed.160 161 Arguments:162 params (`Union[Dict, FrozenDict]`):163 A `PyTree` of model parameters.164 mask (`Union[Dict, FrozenDict]`):165 A `PyTree` with same structure as the `params` tree. The leaves should be booleans, `True` for params166 you want to cast, and should be `False` for those you want to skip167 168 Examples:169 170 ```python171 >>> from diffusers import FlaxUNet2DConditionModel172 173 >>> # load model174 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5")175 >>> # By default, the model params will be in fp32, to cast these to float16176 >>> params = model.to_fp16(params)177 >>> # If you want don't want to cast certain parameters (for example layer norm bias and scale)178 >>> # then pass the mask as follows179 >>> from flax import traverse_util180 181 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5")182 >>> flat_params = traverse_util.flatten_dict(params)183 >>> mask = {184 ... path: (path[-2] != ("LayerNorm", "bias") and path[-2:] != ("LayerNorm", "scale"))185 ... for path in flat_params186 ... }187 >>> mask = traverse_util.unflatten_dict(mask)188 >>> params = model.to_fp16(params, mask)189 ```"""190 return self._cast_floating_to(params, jnp.float16, mask)191 192 def init_weights(self, rng: jax.random.KeyArray) -> Dict:193 raise NotImplementedError(f"init_weights method has to be implemented for {self}")194 195 @classmethod196 def from_pretrained(197 cls,198 pretrained_model_name_or_path: Union[str, os.PathLike],199 dtype: jnp.dtype = jnp.float32,200 *model_args,201 **kwargs,202 ):203 r"""204 Instantiate a pretrained flax model from a pre-trained model configuration.205 206 The warning *Weights from XXX not initialized from pretrained model* means that the weights of XXX do not come207 pretrained with the rest of the model. It is up to you to train those weights with a downstream fine-tuning208 task.209 210 The warning *Weights from XXX not used in YYY* means that the layer XXX is not used by YYY, therefore those211 weights are discarded.212 213 Parameters:214 pretrained_model_name_or_path (`str` or `os.PathLike`):215 Can be either:216 217 - A string, the *model id* of a pretrained model hosted inside a model repo on huggingface.co.218 Valid model ids are namespaced under a user or organization name, like219 `runwayml/stable-diffusion-v1-5`.220 - A path to a *directory* containing model weights saved using [`~ModelMixin.save_pretrained`],221 e.g., `./my_model_directory/`.222 dtype (`jax.numpy.dtype`, *optional*, defaults to `jax.numpy.float32`):223 The data type of the computation. Can be one of `jax.numpy.float32`, `jax.numpy.float16` (on GPUs) and224 `jax.numpy.bfloat16` (on TPUs).225 226 This can be used to enable mixed-precision training or half-precision inference on GPUs or TPUs. If227 specified all the computation will be performed with the given `dtype`.228 229 **Note that this only specifies the dtype of the computation and does not influence the dtype of model230 parameters.**231 232 If you wish to change the dtype of the model parameters, see [`~ModelMixin.to_fp16`] and233 [`~ModelMixin.to_bf16`].234 model_args (sequence of positional arguments, *optional*):235 All remaining positional arguments will be passed to the underlying model's `__init__` method.236 cache_dir (`Union[str, os.PathLike]`, *optional*):237 Path to a directory in which a downloaded pretrained model configuration should be cached if the238 standard cache should not be used.239 force_download (`bool`, *optional*, defaults to `False`):240 Whether or not to force the (re-)download of the model weights and configuration files, overriding the241 cached versions if they exist.242 resume_download (`bool`, *optional*, defaults to `False`):243 Whether or not to delete incompletely received files. Will attempt to resume the download if such a244 file exists.245 proxies (`Dict[str, str]`, *optional*):246 A dictionary of proxy servers to use by protocol or endpoint, e.g., `{'http': 'foo.bar:3128',247 'http://hostname': 'foo.bar:4012'}`. The proxies are used on each request.248 local_files_only(`bool`, *optional*, defaults to `False`):249 Whether or not to only look at local files (i.e., do not try to download the model).250 revision (`str`, *optional*, defaults to `"main"`):251 The specific model version to use. It can be a branch name, a tag name, or a commit id, since we use a252 git-based system for storing models and other artifacts on huggingface.co, so `revision` can be any253 identifier allowed by git.254 from_pt (`bool`, *optional*, defaults to `False`):255 Load the model weights from a PyTorch checkpoint save file.256 kwargs (remaining dictionary of keyword arguments, *optional*):257 Can be used to update the configuration object (after it being loaded) and initiate the model (e.g.,258 `output_attentions=True`). Behaves differently depending on whether a `config` is provided or259 automatically loaded:260 261 - If a configuration is provided with `config`, `**kwargs` will be directly passed to the262 underlying model's `__init__` method (we assume all relevant updates to the configuration have263 already been done)264 - If a configuration is not provided, `kwargs` will be first passed to the configuration class265 initialization function ([`~ConfigMixin.from_config`]). Each key of `kwargs` that corresponds to266 a configuration attribute will be used to override said attribute with the supplied `kwargs`267 value. Remaining keys that do not correspond to any configuration attribute will be passed to the268 underlying model's `__init__` function.269 270 Examples:271 272 ```python273 >>> from diffusers import FlaxUNet2DConditionModel274 275 >>> # Download model and configuration from huggingface.co and cache.276 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("runwayml/stable-diffusion-v1-5")277 >>> # Model was saved using *save_pretrained('./test/saved_model/')* (for example purposes, not runnable).278 >>> model, params = FlaxUNet2DConditionModel.from_pretrained("./test/saved_model/")279 ```"""280 config = kwargs.pop("config", None)281 cache_dir = kwargs.pop("cache_dir", DIFFUSERS_CACHE)282 force_download = kwargs.pop("force_download", False)283 from_pt = kwargs.pop("from_pt", False)284 resume_download = kwargs.pop("resume_download", False)285 proxies = kwargs.pop("proxies", None)286 local_files_only = kwargs.pop("local_files_only", False)287 use_auth_token = kwargs.pop("use_auth_token", None)288 revision = kwargs.pop("revision", None)289 subfolder = kwargs.pop("subfolder", None)290 291 user_agent = {292 "diffusers": __version__,293 "file_type": "model",294 "framework": "flax",295 }296 297 # Load config if we don't provide a configuration298 config_path = config if config is not None else pretrained_model_name_or_path299 model, model_kwargs = cls.from_config(300 config_path,301 cache_dir=cache_dir,302 return_unused_kwargs=True,303 force_download=force_download,304 resume_download=resume_download,305 proxies=proxies,306 local_files_only=local_files_only,307 use_auth_token=use_auth_token,308 revision=revision,309 subfolder=subfolder,310 # model args311 dtype=dtype,312 **kwargs,313 )314 315 # Load model316 pretrained_path_with_subfolder = (317 pretrained_model_name_or_path318 if subfolder is None319 else os.path.join(pretrained_model_name_or_path, subfolder)320 )321 if os.path.isdir(pretrained_path_with_subfolder):322 if from_pt:323 if not os.path.isfile(os.path.join(pretrained_path_with_subfolder, WEIGHTS_NAME)):324 raise EnvironmentError(325 f"Error no file named {WEIGHTS_NAME} found in directory {pretrained_path_with_subfolder} "326 )327 model_file = os.path.join(pretrained_path_with_subfolder, WEIGHTS_NAME)328 elif os.path.isfile(os.path.join(pretrained_path_with_subfolder, FLAX_WEIGHTS_NAME)):329 # Load from a Flax checkpoint330 model_file = os.path.join(pretrained_path_with_subfolder, FLAX_WEIGHTS_NAME)331 # Check if pytorch weights exist instead332 elif os.path.isfile(os.path.join(pretrained_path_with_subfolder, WEIGHTS_NAME)):333 raise EnvironmentError(334 f"{WEIGHTS_NAME} file found in directory {pretrained_path_with_subfolder}. Please load the model"335 " using `from_pt=True`."336 )337 else:338 raise EnvironmentError(339 f"Error no file named {FLAX_WEIGHTS_NAME} or {WEIGHTS_NAME} found in directory "340 f"{pretrained_path_with_subfolder}."341 )342 else:343 try:344 model_file = hf_hub_download(345 pretrained_model_name_or_path,346 filename=FLAX_WEIGHTS_NAME if not from_pt else WEIGHTS_NAME,347 cache_dir=cache_dir,348 force_download=force_download,349 proxies=proxies,350 resume_download=resume_download,351 local_files_only=local_files_only,352 use_auth_token=use_auth_token,353 user_agent=user_agent,354 subfolder=subfolder,355 revision=revision,356 )357 358 except RepositoryNotFoundError:359 raise EnvironmentError(360 f"{pretrained_model_name_or_path} is not a local folder and is not a valid model identifier "361 "listed on 'https://huggingface.co/models'\nIf this is a private repository, make sure to pass a "362 "token having permission to this repo with `use_auth_token` or log in with `huggingface-cli "363 "login`."364 )365 except RevisionNotFoundError:366 raise EnvironmentError(367 f"{revision} is not a valid git identifier (branch name, tag name or commit id) that exists for "368 "this model name. Check the model page at "369 f"'https://huggingface.co/{pretrained_model_name_or_path}' for available revisions."370 )371 except EntryNotFoundError:372 raise EnvironmentError(373 f"{pretrained_model_name_or_path} does not appear to have a file named {FLAX_WEIGHTS_NAME}."374 )375 except HTTPError as err:376 raise EnvironmentError(377 f"There was a specific connection error when trying to load {pretrained_model_name_or_path}:\n"378 f"{err}"379 )380 except ValueError:381 raise EnvironmentError(382 f"We couldn't connect to '{HUGGINGFACE_CO_RESOLVE_ENDPOINT}' to load this model, couldn't find it"383 f" in the cached files and it looks like {pretrained_model_name_or_path} is not the path to a"384 f" directory containing a file named {FLAX_WEIGHTS_NAME} or {WEIGHTS_NAME}.\nCheckout your"385 " internet connection or see how to run the library in offline mode at"386 " 'https://huggingface.co/docs/transformers/installation#offline-mode'."387 )388 except EnvironmentError:389 raise EnvironmentError(390 f"Can't load the model for '{pretrained_model_name_or_path}'. If you were trying to load it from "391 "'https://huggingface.co/models', make sure you don't have a local directory with the same name. "392 f"Otherwise, make sure '{pretrained_model_name_or_path}' is the correct path to a directory "393 f"containing a file named {FLAX_WEIGHTS_NAME} or {WEIGHTS_NAME}."394 )395 396 if from_pt:397 if is_torch_available():398 from .modeling_utils import load_state_dict399 else:400 raise EnvironmentError(401 "Can't load the model in PyTorch format because PyTorch is not installed. "402 "Please, install PyTorch or use native Flax weights."403 )404 405 # Step 1: Get the pytorch file406 pytorch_model_file = load_state_dict(model_file)407 408 # Step 2: Convert the weights409 state = convert_pytorch_state_dict_to_flax(pytorch_model_file, model)410 else:411 try:412 with open(model_file, "rb") as state_f:413 state = from_bytes(cls, state_f.read())414 except (UnpicklingError, msgpack.exceptions.ExtraData) as e:415 try:416 with open(model_file) as f:417 if f.read().startswith("version"):418 raise OSError(419 "You seem to have cloned a repository without having git-lfs installed. Please"420 " install git-lfs and run `git lfs install` followed by `git lfs pull` in the"421 " folder you cloned."422 )423 else:424 raise ValueError from e425 except (UnicodeDecodeError, ValueError):426 raise EnvironmentError(f"Unable to convert {model_file} to Flax deserializable object. ")427 # make sure all arrays are stored as jnp.ndarray428 # NOTE: This is to prevent a bug this will be fixed in Flax >= v0.3.4:429 # https://github.com/google/flax/issues/1261430 state = jax.tree_util.tree_map(lambda x: jax.device_put(x, jax.devices("cpu")[0]), state)431 432 # flatten dicts433 state = flatten_dict(state)434 435 params_shape_tree = jax.eval_shape(model.init_weights, rng=jax.random.PRNGKey(0))436 required_params = set(flatten_dict(unfreeze(params_shape_tree)).keys())437 438 shape_state = flatten_dict(unfreeze(params_shape_tree))439 440 missing_keys = required_params - set(state.keys())441 unexpected_keys = set(state.keys()) - required_params442 443 if missing_keys:444 logger.warning(445 f"The checkpoint {pretrained_model_name_or_path} is missing required keys: {missing_keys}. "446 "Make sure to call model.init_weights to initialize the missing weights."447 )448 cls._missing_keys = missing_keys449 450 for key in state.keys():451 if key in shape_state and state[key].shape != shape_state[key].shape:452 raise ValueError(453 f"Trying to load the pretrained weight for {key} failed: checkpoint has shape "454 f"{state[key].shape} which is incompatible with the model shape {shape_state[key].shape}. "455 )456 457 # remove unexpected keys to not be saved again458 for unexpected_key in unexpected_keys:459 del state[unexpected_key]460 461 if len(unexpected_keys) > 0:462 logger.warning(463 f"Some weights of the model checkpoint at {pretrained_model_name_or_path} were not used when"464 f" initializing {model.__class__.__name__}: {unexpected_keys}\n- This IS expected if you are"465 f" initializing {model.__class__.__name__} from the checkpoint of a model trained on another task or"466 " with another architecture."467 )468 else:469 logger.info(f"All model checkpoint weights were used when initializing {model.__class__.__name__}.\n")470 471 if len(missing_keys) > 0:472 logger.warning(473 f"Some weights of {model.__class__.__name__} were not initialized from the model checkpoint at"474 f" {pretrained_model_name_or_path} and are newly initialized: {missing_keys}\nYou should probably"475 " TRAIN this model on a down-stream task to be able to use it for predictions and inference."476 )477 else:478 logger.info(479 f"All the weights of {model.__class__.__name__} were initialized from the model checkpoint at"480 f" {pretrained_model_name_or_path}.\nIf your task is similar to the task the model of the checkpoint"481 f" was trained on, you can already use {model.__class__.__name__} for predictions without further"482 " training."483 )484 485 return model, unflatten_dict(state)486 487 def save_pretrained(488 self,489 save_directory: Union[str, os.PathLike],490 params: Union[Dict, FrozenDict],491 is_main_process: bool = True,492 ):493 """494 Save a model and its configuration file to a directory, so that it can be re-loaded using the495 `[`~FlaxModelMixin.from_pretrained`]` class method496 497 Arguments:498 save_directory (`str` or `os.PathLike`):499 Directory to which to save. Will be created if it doesn't exist.500 params (`Union[Dict, FrozenDict]`):501 A `PyTree` of model parameters.502 is_main_process (`bool`, *optional*, defaults to `True`):503 Whether the process calling this is the main process or not. Useful when in distributed training like504 TPUs and need to call this function on all processes. In this case, set `is_main_process=True` only on505 the main process to avoid race conditions.506 """507 if os.path.isfile(save_directory):508 logger.error(f"Provided path ({save_directory}) should be a directory, not a file")509 return510 511 os.makedirs(save_directory, exist_ok=True)512 513 model_to_save = self514 515 # Attach architecture to the config516 # Save the config517 if is_main_process:518 model_to_save.save_config(save_directory)519 520 # save model521 output_model_file = os.path.join(save_directory, FLAX_WEIGHTS_NAME)522 with open(output_model_file, "wb") as f:523 model_bytes = to_bytes(params)524 f.write(model_bytes)525 526 logger.info(f"Model weights saved in {output_model_file}")527