Nethermind/Mpt-Instruct-DotNet-XS

1import torch2import torch.nn.functional as F3 4class LPLayerNorm(torch.nn.LayerNorm):5    def __init__(self, normalized_shape, eps=1e-05, elementwise_affine=True, device=None, dtype=None):6        super().__init__(7            normalized_shape=normalized_shape,8            eps=eps,9            elementwise_affine=elementwise_affine,10            device=device,11            dtype=dtype,12        )13 14    def forward(self, x):15        module_device = x.device16        downcast_x = _cast_if_autocast_enabled(x)17        downcast_weight = _cast_if_autocast_enabled(self.weight) if self.weight is not None else self.weight18        downcast_bias = _cast_if_autocast_enabled(self.bias) if self.bias is not None else self.bias19        with torch.autocast(enabled=False, device_type=module_device.type):20            return F.layer_norm(downcast_x, self.normalized_shape, downcast_weight, downcast_bias, self.eps)21 22def _cast_if_autocast_enabled(tensor):23    if torch.is_autocast_enabled():24        if tensor.device.type == 'cuda':25            dtype = torch.get_autocast_gpu_dtype()26        elif tensor.device.type == 'cpu':27            dtype = torch.get_autocast_cpu_dtype()28        else:29            raise NotImplementedError()30        return tensor.to(dtype=dtype)31    return tensor