Aluode/PerceptionLabPortable
0
1# Copyright 2020 The HuggingFace Team. All rights reserved.2#3# Licensed under the Apache License, Version 2.0 (the "License");4# you may not use this file except in compliance with the License.5# You may obtain a copy of the License at6#7# http://www.apache.org/licenses/LICENSE-2.08#9# Unless required by applicable law or agreed to in writing, software10# distributed under the License is distributed on an "AS IS" BASIS,11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.12# See the License for the specific language governing permissions and13# limitations under the License.14 15import functools16import math17from collections import OrderedDict18 19import torch20from torch import Tensor, nn21 22from .integrations.hub_kernels import use_kernel_forward_from_hub23from .utils import logging24from .utils.import_utils import is_torchdynamo_compiling25 26 27logger = logging.get_logger(__name__)28 29 30@use_kernel_forward_from_hub("GeluTanh")31class GELUTanh(nn.Module):32 """33 A fast C implementation of the tanh approximation of the GeLU activation function. See34 https://huggingface.co/papers/1606.08415.35 36 This implementation is equivalent to NewGELU and FastGELU but much faster. However, it is not an exact numerical37 match due to rounding errors.38 """39 40 def __init__(self, use_gelu_tanh_python: bool = False):41 super().__init__()42 if use_gelu_tanh_python:43 self.act = self._gelu_tanh_python44 else:45 self.act = functools.partial(nn.functional.gelu, approximate="tanh")46 47 def _gelu_tanh_python(self, input: Tensor) -> Tensor:48 return input * 0.5 * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * torch.pow(input, 3.0))))49 50 def forward(self, input: Tensor) -> Tensor:51 return self.act(input)52 53 54@use_kernel_forward_from_hub("NewGELU")55class NewGELUActivation(nn.Module):56 """57 Implementation of the GELU activation function currently in Google BERT repo (identical to OpenAI GPT). Also see58 the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.0841559 """60 61 def forward(self, input: Tensor) -> Tensor:62 return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi) * (input + 0.044715 * torch.pow(input, 3.0))))63 64 65@use_kernel_forward_from_hub("GeLU")66class GELUActivation(nn.Module):67 """68 Original Implementation of the GELU activation function in Google BERT repo when initially created. For69 information: OpenAI GPT's GELU is slightly different (and gives slightly different results): 0.5 * x * (1 +70 torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))) This is now written in C in nn.functional71 Also see the Gaussian Error Linear Units paper: https://huggingface.co/papers/1606.0841572 """73 74 def __init__(self, use_gelu_python: bool = False):75 super().__init__()76 if use_gelu_python:77 self.act = self._gelu_python78 else:79 self.act = nn.functional.gelu80 81 def _gelu_python(self, input: Tensor) -> Tensor:82 return input * 0.5 * (1.0 + torch.erf(input / math.sqrt(2.0)))83 84 def forward(self, input: Tensor) -> Tensor:85 return self.act(input)86 87 88@use_kernel_forward_from_hub("SiLU")89class SiLUActivation(nn.Module):90 """91 See Gaussian Error Linear Units (Hendrycks et al., https://arxiv.org/abs/1606.08415) where the SiLU (Sigmoid Linear92 Unit) was originally introduced and coined, and see Sigmoid-Weighted Linear Units for Neural Network Function93 Approximation in Reinforcement Learning (Elfwing et al., https://arxiv.org/abs/1702.03118) and Swish: a Self-Gated94 Activation Function (Ramachandran et al., https://arxiv.org/abs/1710.05941v1) where the SiLU was experimented with95 later.96 """97 98 def forward(self, input: Tensor) -> Tensor:99 return nn.functional.silu(input)100 101 102@use_kernel_forward_from_hub("FastGELU")103class FastGELUActivation(nn.Module):104 """105 Applies GELU approximation that is slower than QuickGELU but more accurate. See: https://github.com/hendrycks/GELUs106 """107 108 def forward(self, input: Tensor) -> Tensor:109 return 0.5 * input * (1.0 + torch.tanh(input * 0.7978845608 * (1.0 + 0.044715 * input * input)))110 111 112@use_kernel_forward_from_hub("QuickGELU")113class QuickGELUActivation(nn.Module):114 """115 Applies GELU approximation that is fast but somewhat inaccurate. See: https://github.com/hendrycks/GELUs116 """117 118 def forward(self, input: Tensor) -> Tensor:119 return input * torch.sigmoid(1.702 * input)120 121 122class ClippedGELUActivation(nn.Module):123 """124 Clip the range of possible GeLU outputs between [min, max]. This is especially useful for quantization purpose, as125 it allows mapping negatives values in the GeLU spectrum. For more information on this trick, please refer to126 https://huggingface.co/papers/2004.09602.127 128 Gaussian Error Linear Unit. Original Implementation of the gelu activation function in Google Bert repo when129 initially created.130 131 For information: OpenAI GPT's gelu is slightly different (and gives slightly different results): 0.5 * x * (1 +132 torch.tanh(math.sqrt(2 / math.pi) * (x + 0.044715 * torch.pow(x, 3)))). See https://huggingface.co/papers/1606.08415133 """134 135 def __init__(self, min: float, max: float):136 if min > max:137 raise ValueError(f"min should be < max (got min: {min}, max: {max})")138 139 super().__init__()140 self.min = min141 self.max = max142 143 def forward(self, x: Tensor) -> Tensor:144 return torch.clip(gelu(x), self.min, self.max)145 146 147class AccurateGELUActivation(nn.Module):148 """149 Applies GELU approximation that is faster than default and more accurate than QuickGELU. See:150 https://github.com/hendrycks/GELUs151 152 Implemented along with MEGA (Moving Average Equipped Gated Attention)153 """154 155 def __init__(self):156 super().__init__()157 self.precomputed_constant = math.sqrt(2 / math.pi)158 159 def forward(self, input: Tensor) -> Tensor:160 return 0.5 * input * (1 + torch.tanh(self.precomputed_constant * (input + 0.044715 * torch.pow(input, 3))))161 162 163class MishActivation(nn.Module):164 """165 See Mish: A Self-Regularized Non-Monotonic Activation Function (Misra., https://huggingface.co/papers/1908.08681). Also166 visit the official repository for the paper: https://github.com/digantamisra98/Mish167 """168 169 def __init__(self):170 super().__init__()171 self.act = nn.functional.mish172 173 def _mish_python(self, input: Tensor) -> Tensor:174 return input * torch.tanh(nn.functional.softplus(input))175 176 def forward(self, input: Tensor) -> Tensor:177 return self.act(input)178 179 180class LinearActivation(nn.Module):181 """182 Applies the linear activation function, i.e. forwarding input directly to output.183 """184 185 def forward(self, input: Tensor) -> Tensor:186 return input187 188 189class LaplaceActivation(nn.Module):190 """191 Applies elementwise activation based on Laplace function, introduced in MEGA as an attention activation. See192 https://huggingface.co/papers/2209.10655193 194 Inspired by squared relu, but with bounded range and gradient for better stability195 """196 197 def forward(self, input, mu=0.707107, sigma=0.282095):198 input = (input - mu).div(sigma * math.sqrt(2.0))199 return 0.5 * (1.0 + torch.erf(input))200 201 202class ReLUSquaredActivation(nn.Module):203 """204 Applies the relu^2 activation introduced in https://huggingface.co/papers/2109.08668v2205 """206 207 def forward(self, input):208 relu_applied = nn.functional.relu(input)209 squared = torch.square(relu_applied)210 return squared211 212 213class ClassInstantier(OrderedDict):214 def __getitem__(self, key):215 content = super().__getitem__(key)216 cls, kwargs = content if isinstance(content, tuple) else (content, {})217 return cls(**kwargs)218 219 220class XIELUActivation(nn.Module):221 """222 Applies the xIELU activation function introduced in https://arxiv.org/abs/2411.13010223 224 If the user has installed the nickjbrowning/XIELU wheel, we import xIELU CUDA225 Otherwise, we emit a single warning and use xIELU Python226 """227 228 def __init__(229 self,230 alpha_p_init=0.8,231 alpha_n_init=0.8,232 beta=0.5,233 eps=-1e-6,234 dtype=torch.bfloat16,235 with_vector_loads=False,236 ):237 super().__init__()238 self.alpha_p = nn.Parameter(torch.log(torch.expm1(torch.tensor(alpha_p_init, dtype=dtype))).unsqueeze(0))239 self.alpha_n = nn.Parameter(240 torch.log(torch.expm1(torch.tensor(alpha_n_init - beta, dtype=dtype))).unsqueeze(0)241 )242 self.register_buffer("beta", torch.tensor(beta, dtype=dtype))243 self.register_buffer("eps", torch.tensor(eps, dtype=dtype))244 self.with_vector_loads = with_vector_loads245 # Temporary until xIELU CUDA fully implemented246 self._beta_scalar = float(self.beta.detach().cpu().float().item())247 self._eps_scalar = float(self.eps.detach().cpu().float().item())248 249 self._xielu_cuda_obj = None250 try:251 import xielu.ops # noqa: F401252 253 self._xielu_cuda_obj = torch.classes.xielu.XIELU()254 msg = "Using experimental xIELU CUDA."255 try:256 from torch._dynamo import allow_in_graph257 258 self._xielu_cuda_fn = allow_in_graph(self._xielu_cuda)259 msg += " Enabled torch._dynamo for xIELU CUDA."260 except Exception as err:261 msg += f" Could not enable torch._dynamo for xIELU ({err}) - this may result in slower performance."262 self._xielu_cuda_fn = self._xielu_cuda263 logger.warning_once(msg)264 except Exception as err:265 logger.warning_once(266 "CUDA-fused xIELU not available (%s) – falling back to a Python version.\n"267 "For CUDA xIELU (experimental), `pip install git+https://github.com/nickjbrowning/XIELU`",268 str(err),269 )270 271 def _xielu_python(self, x: Tensor) -> Tensor:272 alpha_p = nn.functional.softplus(self.alpha_p)273 alpha_n = self.beta + nn.functional.softplus(self.alpha_n)274 return torch.where(275 x > 0,276 alpha_p * x * x + self.beta * x,277 (torch.expm1(torch.min(x, self.eps)) - x) * alpha_n + self.beta * x,278 )279 280 def _xielu_cuda(self, x: Tensor) -> Tensor:281 """Firewall function to prevent torch.compile from seeing .item() calls"""282 original_shape = x.shape283 # CUDA kernel expects 3D tensors, reshape if needed284 while x.dim() < 3:285 x = x.unsqueeze(0)286 if x.dim() > 3:287 x = x.view(-1, 1, x.size(-1))288 if original_shape != x.shape:289 logger.warning_once(290 "Warning: xIELU input tensor expects 3 dimensions but got (shape: %s). Reshaping to (shape: %s).",291 original_shape,292 x.shape,293 )294 result = self._xielu_cuda_obj.forward(295 x,296 self.alpha_p.to(x.dtype),297 self.alpha_n.to(x.dtype),298 # Temporary until xIELU CUDA fully implemented -> self.{beta,eps}.item()299 self._beta_scalar,300 self._eps_scalar,301 self.with_vector_loads,302 )303 return result.view(original_shape)304 305 def forward(self, input: Tensor) -> Tensor:306 if self._xielu_cuda_obj is not None and input.is_cuda:307 if not is_torchdynamo_compiling():308 return self._xielu_cuda_fn(input)309 else:310 logger.warning_once("torch._dynamo is compiling, using Python version of xIELU.")311 return self._xielu_python(input)312 313 314ACT2CLS = {315 "gelu": GELUActivation,316 "gelu_10": (ClippedGELUActivation, {"min": -10, "max": 10}),317 "gelu_fast": FastGELUActivation,318 "gelu_new": NewGELUActivation,319 "gelu_python": (GELUActivation, {"use_gelu_python": True}),320 "gelu_pytorch_tanh": GELUTanh,321 "gelu_python_tanh": (GELUTanh, {"use_gelu_tanh_python": True}),322 "gelu_accurate": AccurateGELUActivation,323 "laplace": LaplaceActivation,324 "leaky_relu": nn.LeakyReLU,325 "linear": LinearActivation,326 "mish": MishActivation,327 "quick_gelu": QuickGELUActivation,328 "relu": nn.ReLU,329 "relu2": ReLUSquaredActivation,330 "relu6": nn.ReLU6,331 "sigmoid": nn.Sigmoid,332 "silu": SiLUActivation,333 "swish": nn.SiLU,334 "tanh": nn.Tanh,335 "prelu": nn.PReLU,336 "xielu": XIELUActivation,337}338ACT2FN = ClassInstantier(ACT2CLS)339 340 341def get_activation(activation_string):342 if activation_string in ACT2FN:343 return ACT2FN[activation_string]344 else:345 raise KeyError(f"function {activation_string} not found in ACT2FN mapping {list(ACT2FN.keys())}")346 347 348# For backwards compatibility with: from activations import gelu_python349gelu_python = get_activation("gelu_python")350gelu_new = get_activation("gelu_new")351gelu = get_activation("gelu")352gelu_fast = get_activation("gelu_fast")353quick_gelu = get_activation("quick_gelu")354silu = get_activation("silu")355mish = get_activation("mish")356linear_act = get_activation("linear")357