RuiTerrty/RemoteSensingChangeDetection-RSCD.HA2F

1# Copyright (c) Meta Platforms, Inc. and affiliates.2#3# This source code is licensed under the Apache License, Version 2.04# found in the LICENSE file in the root directory of this source tree.5 6# Modified from: https://github.com/huggingface/pytorch-image-models/blob/main/timm/models/vision_transformer.py#L103-L1107 8from typing import Union9 10import torch11from torch import Tensor12from torch import nn13 14 15class LayerScale(nn.Module):16    def __init__(17        self,18        dim: int,19        init_values: Union[float, Tensor] = 1e-5,20        inplace: bool = False,21    ) -> None:22        super().__init__()23        self.inplace = inplace24        self.gamma = nn.Parameter(init_values * torch.ones(dim))25 26    def forward(self, x: Tensor) -> Tensor:27        return x.mul_(self.gamma) if self.inplace else x * self.gamma28