Aluode/PerceptionLabPortable

1# Copyright 2020 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7#     http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import math16 17import tensorflow as tf18from packaging.version import parse19 20 21try:22    import tf_keras as keras23except (ModuleNotFoundError, ImportError):24    import keras25 26    if parse(keras.__version__).major > 2:27        raise ValueError(28            "Your currently installed version of Keras is Keras 3, but this is not yet supported in "29            "Transformers. Please install the backwards-compatible tf-keras package with "30            "`pip install tf-keras`."31        )32 33 34def _gelu(x):35    """36    Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when37    initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):38    0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see39    https://huggingface.co/papers/1606.0841540    """41    x = tf.convert_to_tensor(x)42    cdf = 0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0), x.dtype)))43 44    return x * cdf45 46 47def _gelu_new(x):48    """49    Gaussian Error Linear Unit. This is a smoother version of the GELU. Original paper: https://huggingface.co/papers/1606.084150 51    Args:52        x: float Tensor to perform activation53 54    Returns:55        `x` with the GELU activation applied.56    """57    x = tf.convert_to_tensor(x)58    pi = tf.cast(math.pi, x.dtype)59    coeff = tf.cast(0.044715, x.dtype)60    cdf = 0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi) * (x + coeff * tf.pow(x, 3))))61 62    return x * cdf63 64 65def mish(x):66    x = tf.convert_to_tensor(x)67 68    return x * tf.tanh(tf.math.softplus(x))69 70 71def gelu_fast(x):72    x = tf.convert_to_tensor(x)73    coeff1 = tf.cast(0.044715, x.dtype)74    coeff2 = tf.cast(0.7978845608, x.dtype)75 76    return 0.5 * x * (1.0 + tf.tanh(x * coeff2 * (1.0 + coeff1 * x * x)))77 78 79def quick_gelu(x):80    x = tf.convert_to_tensor(x)81    coeff = tf.cast(1.702, x.dtype)82    return x * tf.math.sigmoid(coeff * x)83 84 85def gelu_10(x):86    """87    Clip the range of possible GeLU outputs between [-10, 10]. This is especially useful for quantization purpose, as88    it allows mapping 2 negatives values in the GeLU spectrum. For more information on this trick, please refer to89    https://huggingface.co/papers/2004.0960290 91    Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when92    initially created. For information: OpenAI GPT's gelu is slightly different (and gives slightly different results):93    0.5 * x * (1 + torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) Also see94    https://huggingface.co/papers/1606.08415 :param x: :return:95    """96    return tf.clip_by_value(_gelu(x), -10, 10)97 98 99def glu(x, axis=-1):100    """101    Gated Linear Unit. Implementation as defined in the original paper (see https://huggingface.co/papers/1612.08083), where102    the input `x` is split in two halves across a dimension (`axis`), A and B, returning A * sigmoid(B).103 104    Args:105        `x`: float Tensor to perform activation106        `axis`: dimension across which `x` be split in half107 108    Returns:109        `x` with the GLU activation applied (with its size halved across the dimension `axis`).110    """111    a, b = tf.split(x, 2, axis=axis)112    return a * tf.math.sigmoid(b)113 114 115if parse(tf.version.VERSION) >= parse("2.4"):116 117    def approximate_gelu_wrap(x):118        return keras.activations.gelu(x, approximate=True)119 120    gelu = keras.activations.gelu121    gelu_new = approximate_gelu_wrap122else:123    gelu = _gelu124    gelu_new = _gelu_new125 126 127ACT2FN = {128    "gelu": gelu,129    "gelu_10": gelu_10,130    "gelu_fast": gelu_fast,131    "gelu_new": gelu_new,132    "glu": glu,133    "mish": mish,134    "quick_gelu": quick_gelu,135    "relu": keras.activations.relu,136    "sigmoid": keras.activations.sigmoid,137    "silu": keras.activations.swish,138    "swish": keras.activations.swish,139    "tanh": keras.activations.tanh,140}141 142 143def get_tf_activation(activation_string):144    if activation_string in ACT2FN:145        return ACT2FN[activation_string]146    else:147        raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}")148