andyfriedrich-amd/hipifyplus
02
1[2 {3 "cuda": "\n#include <ATen/cuda/CUDAContext.h>\n\n#include <cuda_runtime.h>\n\nnamespace at { namespace cuda {\n\n/**\n Computes ceil(a / b)\n*/\ntemplate <typename T>\n__host__ __device__ __forceinline__ T ATenCeilDiv(T a, T b) {\n return (a + b - 1) / b;\n}\n\nnamespace {\n\n// Threads per block for our apply kernel\n// FIXME: use occupancy calculator instead\nconstexpr uint32_t AT_APPLY_THREADS_PER_BLOCK = 512;\nconstexpr uint32_t AT_APPLY_BLOCKS_PER_SM = 4;\n\ntemplate <int step = 1>\ninline bool getApplyGrid(uint64_t totalElements, dim3& grid, int64_t curDevice, int max_threads_per_block=AT_APPLY_THREADS_PER_BLOCK) {\n if (curDevice == -1) return false;\n uint64_t numel_per_thread = static_cast<uint64_t>(max_threads_per_block) * static_cast<uint64_t>(step);\n uint64_t numBlocks = ATenCeilDiv(totalElements, numel_per_thread);\n uint64_t maxGridX = at::cuda::getDeviceProperties(curDevice)->maxGridSize[0];\n if (numBlocks > maxGridX)\n numBlocks = maxGridX;\n grid = dim3(numBlocks);\n return true;\n}\n\nconstexpr int getApplyBlocksPerSM() {\n return AT_APPLY_BLOCKS_PER_SM;\n}\n\nconstexpr int getApplyBlockSize() {\n return AT_APPLY_THREADS_PER_BLOCK;\n}\n\ninline dim3 getApplyBlock(int max_threads_per_block=AT_APPLY_THREADS_PER_BLOCK) {\n return dim3(max_threads_per_block);\n}\n\n}\n}} // namespace at::cuda\n\n\n###",4 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/hip\\HIPContext.h>\n\n#include <hip/hip_runtime.h>\n\nnamespace at { namespace hip {\n\n/**\n Computes ceil(a / b)\n*/\ntemplate <typename T>\n__host__ __device__ __forceinline__ T ATenCeilDiv(T a, T b) {\n return (a + b - 1) / b;\n}\n\nnamespace {\n\n// Threads per block for our apply kernel\n// FIXME: use occupancy calculator instead\nconstexpr uint32_t AT_APPLY_THREADS_PER_BLOCK = 512;\nconstexpr uint32_t AT_APPLY_BLOCKS_PER_SM = 4;\n\ntemplate <int step = 1>\ninline bool getApplyGrid(uint64_t totalElements, dim3& grid, int64_t curDevice, int max_threads_per_block=AT_APPLY_THREADS_PER_BLOCK) {\n if (curDevice == -1) return false;\n uint64_t numel_per_thread = static_cast<uint64_t>(max_threads_per_block) * static_cast<uint64_t>(step);\n uint64_t numBlocks = ATenCeilDiv(totalElements, numel_per_thread);\n uint64_t maxGridX = at::cuda::getDeviceProperties(curDevice)->maxGridSize[0];\n if (numBlocks > maxGridX)\n numBlocks = maxGridX;\n grid = dim3(numBlocks);\n return true;\n}\n\nconstexpr int getApplyBlocksPerSM() {\n return AT_APPLY_BLOCKS_PER_SM;\n}\n\nconstexpr int getApplyBlockSize() {\n return AT_APPLY_THREADS_PER_BLOCK;\n}\n\ninline dim3 getApplyBlock(int max_threads_per_block=AT_APPLY_THREADS_PER_BLOCK) {\n return dim3(max_threads_per_block);\n}\n\n}\n}} // namespace at::cuda\n###"5 },6 {7 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <ATen/native/Repeat.h>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/repeat_interleave_native.h>\n#endif\n\ntemplate <typename index_t>\n__global__ static void compute_cuda_kernel(\n index_t* repeat_ptr,\n int64_t* cumsum_ptr,\n index_t* result_ptr,\n int64_t size,\n int64_t result_size) {\n CUDA_KERNEL_ASSERT(result_size == cumsum_ptr[size - 1]);\n int64_t idx = blockIdx.x * blockDim.x + threadIdx.x;\n int64_t stride = (blockDim.x * gridDim.x) / C10_WARP_SIZE;\n int warp_id = idx / C10_WARP_SIZE;\n int tid_in_warp = idx % C10_WARP_SIZE;\n for (int64_t i = warp_id; i < size; i += stride) {\n int64_t end = cumsum_ptr[i];\n index_t repeat = repeat_ptr[i];\n CUDA_KERNEL_ASSERT(repeat >= 0);\n int64_t start = end - repeat;\n for (int64_t j = start + tid_in_warp; j < end; j += C10_WARP_SIZE) {\n result_ptr[j] = i;\n }\n }\n}\n\ntemplate <typename index_t>\nstatic void compute_cuda(\n index_t* repeat_ptr,\n int64_t* cumsum_ptr,\n index_t* result_ptr,\n int64_t size,\n int64_t result_size) {\n int64_t block = 512;\n int64_t warps_per_block = block / at::cuda::warp_size();\n int64_t grid =\n std::min<int64_t>((size + warps_per_block - 1) / warps_per_block, 2048L);\n\n compute_cuda_kernel<<<grid, block, 0, at::cuda::getCurrentCUDAStream()>>>(\n repeat_ptr, cumsum_ptr, result_ptr, size, result_size);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\nnamespace at::native {\n\nTensor repeat_interleave_cuda(\n const Tensor& repeat,\n c10::optional<int64_t> output_size) {\n Tensor output;\n AT_DISPATCH_INDEX_TYPES(\n repeat.scalar_type(), \"repeat_interleave_cuda\", [&]() {\n output = repeat_interleave_common<index_t, compute_cuda<index_t>>(\n repeat, output_size);\n });\n return output;\n}\n\n} // namespace at::native\n\n\n###",8 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n#include <ATen/hip\\HIPContext.h>\n#include <ATen/native/Repeat.h>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/repeat_interleave_native.h>\n#endif\n\ntemplate <typename index_t>\n__global__ static void compute_hip_kernel(\n index_t* repeat_ptr,\n int64_t* cumsum_ptr,\n index_t* result_ptr,\n int64_t size,\n int64_t result_size) {\n CUDA_KERNEL_ASSERT(result_size == cumsum_ptr[size - 1]);\n int64_t idx = blockIdx.x * blockDim.x + threadIdx.x;\n int64_t stride = (blockDim.x * gridDim.x) / C10_WARP_SIZE;\n int warp_id = idx / C10_WARP_SIZE;\n int tid_in_warp = idx % C10_WARP_SIZE;\n for (int64_t i = warp_id; i < size; i += stride) {\n int64_t end = cumsum_ptr[i];\n index_t repeat = repeat_ptr[i];\n CUDA_KERNEL_ASSERT(repeat >= 0);\n int64_t start = end - repeat;\n for (int64_t j = start + tid_in_warp; j < end; j += C10_WARP_SIZE) {\n result_ptr[j] = i;\n }\n }\n}\n\ntemplate <typename index_t>\nstatic void compute_hip(\n index_t* repeat_ptr,\n int64_t* cumsum_ptr,\n index_t* result_ptr,\n int64_t size,\n int64_t result_size) {\n int64_t block = 512;\n int64_t warps_per_block = block / at::cuda::warp_size();\n int64_t grid =\n std::min<int64_t>((size + warps_per_block - 1) / warps_per_block, 2048L);\n\n hipLaunchKernelGGL(( compute_hip_kernel), dim3(grid), dim3(block), 0, at::hip::getCurrentHIPStream(), \n repeat_ptr, cumsum_ptr, result_ptr, size, result_size);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\nnamespace at::native {\n\nTensor repeat_interleave_hip(\n const Tensor& repeat,\n c10::optional<int64_t> output_size) {\n Tensor output;\n AT_DISPATCH_INDEX_TYPES(\n repeat.scalar_type(), \"repeat_interleave_hip\", [&]() {\n output = repeat_interleave_common<index_t, compute_hip<index_t>>(\n repeat, output_size);\n });\n return output;\n}\n\n} // namespace at::native\n###"9 },10 {11 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char scaled_modified_bessel_k0_name[] = \"scaled_modified_bessel_k0_forward\";\n\n void scaled_modified_bessel_k0_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k0_cuda\", [&]() {\n jitted_gpu_kernel<scaled_modified_bessel_k0_name, scalar_t, scalar_t, 1>(iterator, scaled_modified_bessel_k0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k0_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return scaled_modified_bessel_k0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_scaled_modified_bessel_k0_stub, &scaled_modified_bessel_k0_kernel_cuda);\n} // namespace at::native\n\n\n###",12 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char scaled_modified_bessel_k0_name[] = \"scaled_modified_bessel_k0_forward\";\n\n void scaled_modified_bessel_k0_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k0_hip\", [&]() {\n jitted_gpu_kernel<scaled_modified_bessel_k0_name, scalar_t, scalar_t, 1>(iterator, scaled_modified_bessel_k0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k0_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return scaled_modified_bessel_k0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_scaled_modified_bessel_k0_stub, &scaled_modified_bessel_k0_kernel_hip);\n} // namespace at::native\n###"13 },14 {15 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char scaled_modified_bessel_k1_name[] = \"scaled_modified_bessel_k1_forward\";\n\n void scaled_modified_bessel_k1_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k1_cuda\", [&]() {\n jitted_gpu_kernel<scaled_modified_bessel_k1_name, scalar_t, scalar_t, 1>(iterator, scaled_modified_bessel_k1_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k1_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return scaled_modified_bessel_k1_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_scaled_modified_bessel_k1_stub, &scaled_modified_bessel_k1_kernel_cuda);\n} // namespace at::native\n\n\n###",16 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char scaled_modified_bessel_k1_name[] = \"scaled_modified_bessel_k1_forward\";\n\n void scaled_modified_bessel_k1_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k1_hip\", [&]() {\n jitted_gpu_kernel<scaled_modified_bessel_k1_name, scalar_t, scalar_t, 1>(iterator, scaled_modified_bessel_k1_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"scaled_modified_bessel_k1_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return scaled_modified_bessel_k1_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_scaled_modified_bessel_k1_stub, &scaled_modified_bessel_k1_kernel_hip);\n} // namespace at::native\n###"17 },18 {19 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char shifted_chebyshev_polynomial_t_name[] = \"shifted_chebyshev_polynomial_t_forward\";\n\n void shifted_chebyshev_polynomial_t_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_t_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_t_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_t_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_t_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_t_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // shifted_chebyshev_polynomial_t_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(shifted_chebyshev_polynomial_t_stub, &shifted_chebyshev_polynomial_t_kernel_cuda);\n} // namespace at::native\n\n\n###",20 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char shifted_chebyshev_polynomial_t_name[] = \"shifted_chebyshev_polynomial_t_forward\";\n\n void shifted_chebyshev_polynomial_t_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_t_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_t_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_t_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_t_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_t_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // shifted_chebyshev_polynomial_t_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(shifted_chebyshev_polynomial_t_stub, &shifted_chebyshev_polynomial_t_kernel_hip);\n} // namespace at::native\n###"21 },22 {23 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char shifted_chebyshev_polynomial_u_name[] = \"shifted_chebyshev_polynomial_u_forward\";\n\n void shifted_chebyshev_polynomial_u_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_u_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_u_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_u_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_u_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_u_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // shifted_chebyshev_polynomial_u_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(shifted_chebyshev_polynomial_u_stub, &shifted_chebyshev_polynomial_u_kernel_cuda);\n} // namespace at::native\n\n\n###",24 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char shifted_chebyshev_polynomial_u_name[] = \"shifted_chebyshev_polynomial_u_forward\";\n\n void shifted_chebyshev_polynomial_u_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_u_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_u_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_u_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_u_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_u_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // shifted_chebyshev_polynomial_u_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(shifted_chebyshev_polynomial_u_stub, &shifted_chebyshev_polynomial_u_kernel_hip);\n} // namespace at::native\n###"25 },26 {27 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_CUDA char shifted_chebyshev_polynomial_v_name[] = \"shifted_chebyshev_polynomial_v_forward\";\n\nvoid shifted_chebyshev_polynomial_v_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_v_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_v_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_v_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_v_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_v_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n} // shifted_chebyshev_polynomial_v_kernel_cuda\n\n} // namespace (anonymous)\n\nREGISTER_DISPATCH(shifted_chebyshev_polynomial_v_stub, &shifted_chebyshev_polynomial_v_kernel_cuda);\n} // namespace at::native\n\n\n###",28 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_HIP char shifted_chebyshev_polynomial_v_name[] = \"shifted_chebyshev_polynomial_v_forward\";\n\nvoid shifted_chebyshev_polynomial_v_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_v_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_v_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_v_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_v_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_v_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n} // shifted_chebyshev_polynomial_v_kernel_hip\n\n} // namespace (anonymous)\n\nREGISTER_DISPATCH(shifted_chebyshev_polynomial_v_stub, &shifted_chebyshev_polynomial_v_kernel_hip);\n} // namespace at::native\n###"29 },30 {31 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char shifted_chebyshev_polynomial_w_name[] = \"shifted_chebyshev_polynomial_w_forward\";\n\n void shifted_chebyshev_polynomial_w_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_w_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_w_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_w_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_w_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_w_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // shifted_chebyshev_polynomial_w_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(shifted_chebyshev_polynomial_w_stub, &shifted_chebyshev_polynomial_w_kernel_cuda);\n} // namespace at::native\n\n\n###",32 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char shifted_chebyshev_polynomial_w_name[] = \"shifted_chebyshev_polynomial_w_forward\";\n\n void shifted_chebyshev_polynomial_w_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_w_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<shifted_chebyshev_polynomial_w_name, scalar_t, scalar_t>(iterator, shifted_chebyshev_polynomial_w_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"shifted_chebyshev_polynomial_w_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return shifted_chebyshev_polynomial_w_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // shifted_chebyshev_polynomial_w_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(shifted_chebyshev_polynomial_w_stub, &shifted_chebyshev_polynomial_w_kernel_hip);\n} // namespace at::native\n###"33 },34 {35 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <thrust/execution_policy.h>\n#include <thrust/sort.h>\n\nnamespace at::native {\n\nstd::vector<int64_t> infer_dense_strides_dim_last(const Tensor & self, int64_t dim) {\n int64_t ndim = self.dim();\n // sort the strides in descending order according to its value,\n // keeping dim the last.\n std::vector<int64_t> strides = self.strides().vec();\n strides[dim] = -1;\n std::vector<int64_t> original_dim(ndim);\n for (int64_t i = 0; i < ndim; i++) {\n original_dim[i] = i;\n }\n thrust::stable_sort_by_key(\n thrust::host, strides.data(), strides.data() + ndim, original_dim.data(),\n thrust::greater<int64_t>()\n );\n // generate contiguous strides on permuted dims\n std::vector<int64_t> new_strides(ndim);\n std::vector<int64_t> new_strides_unsort(ndim);\n int64_t cumprod = 1;\n for (int64_t i = 0; i < ndim; i++) {\n new_strides[ndim - 1 - i] = cumprod;\n cumprod *= self.sizes()[original_dim[ndim - 1 - i]];\n }\n // unsort new strides\n for (int64_t i = 0; i < ndim; i++) {\n new_strides_unsort[original_dim[i]] = new_strides[i];\n }\n return new_strides_unsort;\n}\n\n} // namespace at::native\n\n\n###",36 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <thrust/execution_policy.h>\n#include <thrust/sort.h>\n\nnamespace at::native {\n\nstd::vector<int64_t> infer_dense_strides_dim_last(const Tensor & self, int64_t dim) {\n int64_t ndim = self.dim();\n // sort the strides in descending order according to its value,\n // keeping dim the last.\n std::vector<int64_t> strides = self.strides().vec();\n strides[dim] = -1;\n std::vector<int64_t> original_dim(ndim);\n for (int64_t i = 0; i < ndim; i++) {\n original_dim[i] = i;\n }\n thrust::stable_sort_by_key(\n thrust::host, strides.data(), strides.data() + ndim, original_dim.data(),\n thrust::greater<int64_t>()\n );\n // generate contiguous strides on permuted dims\n std::vector<int64_t> new_strides(ndim);\n std::vector<int64_t> new_strides_unsort(ndim);\n int64_t cumprod = 1;\n for (int64_t i = 0; i < ndim; i++) {\n new_strides[ndim - 1 - i] = cumprod;\n cumprod *= self.sizes()[original_dim[ndim - 1 - i]];\n }\n // unsort new strides\n for (int64_t i = 0; i < ndim; i++) {\n new_strides_unsort[original_dim[i]] = new_strides[i];\n }\n return new_strides_unsort;\n}\n\n} // namespace at::native\n###"37 },38 {39 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <c10/util/Exception.h>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/sspaddmm_native.h>\n#endif\n\nnamespace at::native {\n// sparse, sparse, sparse, dense, real, real -> sparse\nTensor& _sspaddmm_out_only_sparse_cuda(const Tensor& self,\n const Tensor& mat1, const Tensor& mat2, const Scalar& beta, const Scalar& alpha, Tensor& result) {\n AT_ERROR(\"tensor.sspaddmm(...) can only be called on sparse tensors\");\n}\nTensor& _sspaddmm_out_cuda(const Tensor& self,\n const Tensor& mat1, const Tensor& mat2, const Scalar& beta, const Scalar& alpha, Tensor& result) {\n AT_ERROR(\"NYI: CUDA sspaddmm is not implemented\");\n}\n} // namespace at::native\n\n\n###",40 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <c10/util/Exception.h>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/sspaddmm_native.h>\n#endif\n\nnamespace at::native {\n// sparse, sparse, sparse, dense, real, real -> sparse\nTensor& _sspaddmm_out_only_sparse_hip(const Tensor& self,\n const Tensor& mat1, const Tensor& mat2, const Scalar& beta, const Scalar& alpha, Tensor& result) {\n AT_ERROR(\"tensor.sspaddmm(...) can only be called on sparse tensors\");\n}\nTensor& _sspaddmm_out_hip(const Tensor& self,\n const Tensor& mat1, const Tensor& mat2, const Scalar& beta, const Scalar& alpha, Tensor& result) {\n AT_ERROR(\"NYI: HIP sspaddmm is not implemented\");\n}\n} // namespace at::native\n###"41 },42 {43 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char spherical_bessel_j0_name[] = \"spherical_bessel_j0_forward\";\n\n void spherical_bessel_j0_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"spherical_bessel_j0_cuda\", [&]() {\n jitted_gpu_kernel<spherical_bessel_j0_name, scalar_t, scalar_t, 1>(iterator, spherical_bessel_j0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"spherical_bessel_j0_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return spherical_bessel_j0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_spherical_bessel_j0_stub, &spherical_bessel_j0_kernel_cuda);\n} // namespace at::native\n\n\n###",44 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char spherical_bessel_j0_name[] = \"spherical_bessel_j0_forward\";\n\n void spherical_bessel_j0_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"spherical_bessel_j0_hip\", [&]() {\n jitted_gpu_kernel<spherical_bessel_j0_name, scalar_t, scalar_t, 1>(iterator, spherical_bessel_j0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"spherical_bessel_j0_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return spherical_bessel_j0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_spherical_bessel_j0_stub, &spherical_bessel_j0_kernel_hip);\n} // namespace at::native\n###"45 },46 {47 "cuda": "\n#pragma once\n\n#include <ATen/core/TensorBase.h>\n#include <ATen/cuda/detail/TensorInfo.cuh>\n#include <ATen/native/CanUse32BitIndexMath.h>\n\nnamespace at {\nnamespace cuda {\nnamespace detail {\n\nTORCH_CUDA_CU_API bool maybeOverlappingIndices(const at::TensorBase &t);\nusing at::native::canUse32BitIndexMath;\n\ntemplate <typename scalar, typename IndexType>\nTensorInfo<scalar, IndexType>\ngetTensorInfo(const at::TensorBase &t) {\n IndexType sz[MAX_TENSORINFO_DIMS];\n IndexType st[MAX_TENSORINFO_DIMS];\n\n int dims = t.dim();\n for (int i = 0; i < dims; ++i) {\n sz[i] = t.size(i);\n st[i] = t.stride(i);\n }\n\n return TensorInfo<scalar, IndexType>(\n t.data_ptr<scalar>(), dims, sz, st);\n}\n\n} // detail\n} // cuda\n} // at\n\n\n###",48 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <ATen/core/TensorBase.h>\n#include <ATen/hip/detail\\TensorInfo.cuh>\n#include <ATen/native/CanUse32BitIndexMath.h>\n\nnamespace at {\nnamespace hip {\nnamespace detail {\n\nTORCH_HIP_CU_API bool maybeOverlappingIndices(const at::TensorBase &t);\nusing at::native::canUse32BitIndexMath;\n\ntemplate <typename scalar, typename IndexType>\nTensorInfo<scalar, IndexType>\ngetTensorInfo(const at::TensorBase &t) {\n IndexType sz[MAX_TENSORINFO_DIMS];\n IndexType st[MAX_TENSORINFO_DIMS];\n\n int dims = t.dim();\n for (int i = 0; i < dims; ++i) {\n sz[i] = t.size(i);\n st[i] = t.stride(i);\n }\n\n return TensorInfo<scalar, IndexType>(\n t.data_ptr<scalar>(), dims, sz, st);\n}\n\n} // detail\n} // cuda\n} // at\n###"49 },50 {51 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n#include <c10/util/BFloat16-math.h>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid nextafter_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND(kBFloat16, iter.common_dtype(), \"nextafter_cuda\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return std::nextafter(a, b);\n });\n });\n}\n\nvoid heaviside_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_ALL_TYPES_AND3(kHalf, kBool, kBFloat16, iter.dtype(), \"heaviside_cuda\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return a == 0 ? b : static_cast<scalar_t>(a > 0);\n });\n });\n}\n\nREGISTER_DISPATCH(nextafter_stub, &nextafter_kernel_cuda);\nREGISTER_DISPATCH(heaviside_stub, &heaviside_kernel_cuda);\n\n} // namespace at::native\n\n\n###",52 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n#include <c10/util/BFloat16-math.h>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid nextafter_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND(kBFloat16, iter.common_dtype(), \"nextafter_hip\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return std::nextafter(a, b);\n });\n });\n}\n\nvoid heaviside_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_ALL_TYPES_AND3(kHalf, kBool, kBFloat16, iter.dtype(), \"heaviside_hip\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return a == 0 ? b : static_cast<scalar_t>(a > 0);\n });\n });\n}\n\nREGISTER_DISPATCH(nextafter_stub, &nextafter_kernel_hip);\nREGISTER_DISPATCH(heaviside_stub, &heaviside_kernel_hip);\n\n} // namespace at::native\n###"53 },54 {55 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char acosh_name[] = \"acosh_impl\";\n#endif\n\nvoid acosh_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if(at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto acosh_string = jiterator_stringify(\n template <typename T>\n T acosh_impl(T a) {\n return std::acosh(a);\n }\n );\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"acosh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/ acosh_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 1>(iter, acosh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"acosh_name\", [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::acosh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half, ScalarType::BFloat16,\n common_dtype, \"acosh_cuda\",\n [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::acosh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(acosh_stub, &acosh_kernel_cuda);\n\n} // namespace at::native\n\n\n###",56 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char acosh_name[] = \"acosh_impl\";\n#endif\n\nvoid acosh_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if(at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto acosh_string = jiterator_stringify(\n template <typename T>\n T acosh_impl(T a) {\n return std::acosh(a);\n }\n );\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"acosh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/ acosh_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 1>(iter, acosh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"acosh_name\", [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::acosh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half, ScalarType::BFloat16,\n common_dtype, \"acosh_hip\",\n [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::acosh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(acosh_stub, &acosh_kernel_hip);\n\n} // namespace at::native\n###"57 },58 {59 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char acos_name[] = \"acos_impl\";\n#endif\nvoid acos_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto acos_string = jiterator_stringify(\n template <typename T> T acos_impl(T a) { return std::acos(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"acos_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/acos_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, acos_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"acos_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::acos(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"acos_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::acos(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(acos_stub, &acos_kernel_cuda);\n\n} // namespace at::native\n\n\n###",60 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char acos_name[] = \"acos_impl\";\n#endif\nvoid acos_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto acos_string = jiterator_stringify(\n template <typename T> T acos_impl(T a) { return std::acos(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"acos_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/acos_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, acos_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"acos_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::acos(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"acos_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::acos(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(acos_stub, &acos_kernel_hip);\n\n} // namespace at::native\n###"61 },62 {63 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char asinh_name[] = \"asinh_impl\";\n#endif\n\nvoid asinh_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto asinh_string = jiterator_stringify(\n template <typename T> T asinh_impl(T a) { return std::asinh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asinh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/asinh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, asinh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asinh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::asinh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"asinh_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::asinh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(asinh_stub, &asinh_kernel_cuda);\n\n} // namespace at::native\n\n\n###",64 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char asinh_name[] = \"asinh_impl\";\n#endif\n\nvoid asinh_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto asinh_string = jiterator_stringify(\n template <typename T> T asinh_impl(T a) { return std::asinh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asinh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/asinh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, asinh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asinh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::asinh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"asinh_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::asinh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(asinh_stub, &asinh_kernel_hip);\n\n} // namespace at::native\n###"65 },66 {67 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char asin_name[] = \"asin_impl\";\n#endif\n\nvoid asin_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto asin_string = jiterator_stringify(\n template <typename T> T asin_impl(T a) { return std::asin(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asin_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/asin_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, asin_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asin_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::asin(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, common_dtype, \"asin_cuda\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::asin(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(asin_stub, &asin_kernel_cuda);\n\n} // namespace at::native\n\n\n###",68 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char asin_name[] = \"asin_impl\";\n#endif\n\nvoid asin_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto asin_string = jiterator_stringify(\n template <typename T> T asin_impl(T a) { return std::asin(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asin_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/asin_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, asin_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"asin_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::asin(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, common_dtype, \"asin_hip\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::asin(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(asin_stub, &asin_kernel_hip);\n\n} // namespace at::native\n###"69 },70 {71 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char atanh_name[] = \"atanh_impl\";\n#endif\n\nvoid atanh_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto atanh_string = jiterator_stringify(\n template <typename T> T atanh_impl(T a) { return std::atanh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"atanh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/atanh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, atanh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"atanh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::atanh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"atanh_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::atanh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(atanh_stub, &atanh_kernel_cuda);\n\n} // namespace at::native\n\n\n###",72 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char atanh_name[] = \"atanh_impl\";\n#endif\n\nvoid atanh_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto atanh_string = jiterator_stringify(\n template <typename T> T atanh_impl(T a) { return std::atanh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"atanh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/atanh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, atanh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"atanh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::atanh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"atanh_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::atanh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(atanh_stub, &atanh_kernel_hip);\n\n} // namespace at::native\n###"73 },74 {75 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char atan_name[] = \"atan_impl\";\n#endif\n\nvoid atan_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto atan_string = jiterator_stringify(\n template <typename T>\n T atan_impl(T a) {\n return std::atan(a);\n }\n );\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"atan_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/ atan_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 1>(iter, atan_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"atan_name\", [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::atan(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half, ScalarType::BFloat16,\n common_dtype, \"atan_cuda\",\n [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::atan(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(atan_stub, &atan_kernel_cuda);\n\n} // namespace at::native\n\n\n###",76 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char atan_name[] = \"atan_impl\";\n#endif\n\nvoid atan_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto atan_string = jiterator_stringify(\n template <typename T>\n T atan_impl(T a) {\n return std::atan(a);\n }\n );\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"atan_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/ atan_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 1>(iter, atan_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, common_dtype, \"atan_name\", [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::atan(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half, ScalarType::BFloat16,\n common_dtype, \"atan_hip\",\n [&]() {\n gpu_kernel(iter, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::atan(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(atan_stub, &atan_kernel_hip);\n\n} // namespace at::native\n###"77 },78 {79 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char cosh_name[] = \"cosh_impl\";\n#endif\n\nvoid cosh_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto cosh_string = jiterator_stringify(\n template <typename T> T cosh_impl(T a) { return std::cosh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cosh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/cosh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, cosh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cosh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::cosh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"cosh_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::cosh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(cosh_stub, &cosh_kernel_cuda);\n\n} // namespace at::native\n\n\n###",80 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char cosh_name[] = \"cosh_impl\";\n#endif\n\nvoid cosh_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto cosh_string = jiterator_stringify(\n template <typename T> T cosh_impl(T a) { return std::cosh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cosh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/cosh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, cosh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cosh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::cosh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"cosh_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::cosh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(cosh_stub, &cosh_kernel_hip);\n\n} // namespace at::native\n###"81 },82 {83 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char cos_name[] = \"cos_impl\";\n#endif // AT_USE_JITERATOR()\n\nvoid cos_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto cos_string = jiterator_stringify(\n template <typename T> T cos_impl(T a) { return std::cos(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cos_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/cos_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, cos_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cos_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::cos(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"cos_cuda\",\n [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t { return ::cos(a); });\n });\n }\n}\n\nREGISTER_DISPATCH(cos_stub, &cos_kernel_cuda);\n\n} // namespace at::native\n\n\n###",84 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char cos_name[] = \"cos_impl\";\n#endif // AT_USE_JITERATOR()\n\nvoid cos_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto cos_string = jiterator_stringify(\n template <typename T> T cos_impl(T a) { return std::cos(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cos_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/cos_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, cos_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"cos_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::cos(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"cos_hip\",\n [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t { return ::cos(a); });\n });\n }\n}\n\nREGISTER_DISPATCH(cos_stub, &cos_kernel_hip);\n\n} // namespace at::native\n###"85 },86 {87 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char sinh_name[] = \"sinh_impl\";\n#endif\n\nvoid sinh_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto sinh_string = jiterator_stringify(\n template <typename T> T sinh_impl(T a) { return std::sinh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sinh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/sinh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, sinh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sinh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::sinh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"sinh_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::sinh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(sinh_stub, &sinh_kernel_cuda);\n\n} // namespace at::native\n\n\n###",88 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char sinh_name[] = \"sinh_impl\";\n#endif\n\nvoid sinh_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto sinh_string = jiterator_stringify(\n template <typename T> T sinh_impl(T a) { return std::sinh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sinh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/sinh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, sinh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sinh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::sinh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"sinh_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::sinh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(sinh_stub, &sinh_kernel_hip);\n\n} // namespace at::native\n###"89 },90 {91 "cuda": "\n#pragma once\n#include <assert.h>\n#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)\n#include <cuda_runtime.h>\n#endif\nnamespace at {\nnamespace cuda {\nnamespace detail {\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\ntemplate <typename Value>\nstruct DivMod {\n Value div, mod;\n C10_HOST_DEVICE DivMod(Value div, Value mod) : div(div), mod(mod) { }\n};\n\n\ntemplate <typename Value>\nstruct IntDivider {\n IntDivider() = default;\n IntDivider(Value d) : divisor(d) { }\n C10_HOST_DEVICE inline Value div(Value n) const { return n / divisor; }\n C10_HOST_DEVICE inline Value mod(Value n) const { return n % divisor; }\n C10_HOST_DEVICE inline DivMod<Value> divmod(Value n) const {\n return DivMod<Value>(n / divisor, n % divisor);\n }\n Value divisor;\n};\n\ntemplate <>\nstruct IntDivider<unsigned int> {\n static_assert(sizeof(unsigned int) == 4, \"Assumes 32-bit unsigned int.\");\n IntDivider() = default;\n IntDivider(unsigned int d) : divisor(d) {\n assert(divisor >= 1 && divisor <= INT32_MAX);\n \n for (shift = 0; shift < 32; shift++) if ((1U << shift) >= divisor) break;\n uint64_t one = 1;\n uint64_t magic = ((one << 32) * ((one << shift) - divisor)) / divisor + 1;\n m1 = magic;\n assert(m1 > 0 && m1 == magic); \n }\n C10_HOST_DEVICE inline unsigned int div(unsigned int n) const {\n#if defined(__CUDA_ARCH__) || defined(__HIP_DEVICE_COMPILE__)\n \n \n unsigned int t = __umulhi(n, m1);\n return (t + n) >> shift;\n#else\n \n uint64_t t = ((uint64_t) n * m1) >> 32;\n return (t + n) >> shift;\n#endif\n }\n C10_HOST_DEVICE inline unsigned int mod(unsigned int n) const {\n return n - div(n) * divisor;\n }\n C10_HOST_DEVICE inline DivMod<unsigned int> divmod(unsigned int n) const {\n unsigned int q = div(n);\n return DivMod<unsigned int>(q, n - q * divisor);\n }\n unsigned int divisor; \n unsigned int m1; \n unsigned int shift; \n};\n}}} \n\n###",92 "hip": " \n#pragma once\n#include <assert.h>\n#if defined(__HIP_ARCH__) || defined(__HIP_DEVICE_COMPILE__)\n#include <hip/hip_runtime.h>\n#endif\nnamespace at {\nnamespace hip {\nnamespace detail {\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\ntemplate <typename Value>\nstruct DivMod {\n Value div, mod;\n C10_HOST_DEVICE DivMod(Value div, Value mod) : div(div), mod(mod) { }\n};\n\n\ntemplate <typename Value>\nstruct IntDivider {\n IntDivider() = default;\n IntDivider(Value d) : divisor(d) { }\n C10_HOST_DEVICE inline Value div(Value n) const { return n / divisor; }\n C10_HOST_DEVICE inline Value mod(Value n) const { return n % divisor; }\n C10_HOST_DEVICE inline DivMod<Value> divmod(Value n) const {\n return DivMod<Value>(n / divisor, n % divisor);\n }\n Value divisor;\n};\n\ntemplate <>\nstruct IntDivider<unsigned int> {\n static_assert(sizeof(unsigned int) == 4, \"Assumes 32-bit unsigned int.\");\n IntDivider() = default;\n IntDivider(unsigned int d) : divisor(d) {\n assert(divisor >= 1 && divisor <= INT32_MAX);\n \n for (shift = 0; shift < 32; shift++) if ((1U << shift) >= divisor) break;\n uint64_t one = 1;\n uint64_t magic = ((one << 32) * ((one << shift) - divisor)) / divisor + 1;\n m1 = magic;\n assert(m1 > 0 && m1 == magic); \n }\n C10_HOST_DEVICE inline unsigned int div(unsigned int n) const {\n#if defined(__HIP_ARCH__) || defined(__HIP_DEVICE_COMPILE__)\n \n \n unsigned int t = __umulhi(n, m1);\n return (t + n) >> shift;\n#else\n \n uint64_t t = ((uint64_t) n * m1) >> 32;\n return (t + n) >> shift;\n#endif\n }\n C10_HOST_DEVICE inline unsigned int mod(unsigned int n) const {\n return n - div(n) * divisor;\n }\n C10_HOST_DEVICE inline DivMod<unsigned int> divmod(unsigned int n) const {\n unsigned int q = div(n);\n return DivMod<unsigned int>(q, n - q * divisor);\n }\n unsigned int divisor; \n unsigned int m1; \n unsigned int shift; \n};\n}}} ###"93 },94 {95 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char sin_name[] = \"sin_impl\";\n#endif\n\nvoid sin_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto sin_string = jiterator_stringify(\n template <typename T> T sin_impl(T a) { return std::sin(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sin_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/sin_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, sin_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sin_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::sin(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"sin_cuda\",\n [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t { return ::sin(a); });\n });\n }\n}\n\nREGISTER_DISPATCH(sin_stub, &sin_kernel_cuda);\n\n} // namespace at::native\n\n\n###",96 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char sin_name[] = \"sin_impl\";\n#endif\n\nvoid sin_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n#if AT_USE_JITERATOR()\n static const auto sin_string = jiterator_stringify(\n template <typename T> T sin_impl(T a) { return std::sin(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sin_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/sin_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, sin_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"sin_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::sin(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"sin_hip\",\n [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t { return ::sin(a); });\n });\n }\n}\n\nREGISTER_DISPATCH(sin_stub, &sin_kernel_hip);\n\n} // namespace at::native\n###"97 },98 {99 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char tanh_name[] = \"tanh_impl\";\n#endif\n\nvoid tanh_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto tanh_string = jiterator_stringify(\n template <typename T> T tanh_impl(T a) { return std::tanh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tanh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/tanh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, tanh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tanh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::tanh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"tanh_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::tanh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(tanh_stub, &tanh_kernel_cuda);\n\n} // namespace at::native\n\n\n###",100 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char tanh_name[] = \"tanh_impl\";\n#endif\n\nvoid tanh_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto tanh_string = jiterator_stringify(\n template <typename T> T tanh_impl(T a) { return std::tanh(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tanh_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/tanh_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, tanh_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tanh_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::tanh(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"tanh_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return ::tanh(a);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(tanh_stub, &tanh_kernel_hip);\n\n} // namespace at::native\n###"101 },102 {103 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_CUDA char tan_name[] = \"tan_impl\";\n#endif\n\nvoid tan_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto tan_string = jiterator_stringify(\n template <typename T> T tan_impl(T a) { return std::tan(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tan_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/tan_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, tan_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tan_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::tan(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"tan_cuda\",\n [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t { return ::tan(a); });\n });\n }\n}\n\nREGISTER_DISPATCH(tan_stub, &tan_kernel_cuda);\n\n} // namespace at::native\n\n\n###",104 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/OpMathType.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <limits>\n\nnamespace at::native {\n\n#if 0 && AT_USE_JITERATOR()\nCONSTEXPR_EXCEPT_WIN_HIP char tan_name[] = \"tan_impl\";\n#endif\n\nvoid tan_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (at::isComplexType(common_dtype)) {\n // Disabled due to accuracy issues\n#if 0 && AT_USE_JITERATOR()\n static const auto tan_string = jiterator_stringify(\n template <typename T> T tan_impl(T a) { return std::tan(a); });\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tan_name\", [&]() {\n jitted_gpu_kernel<\n /*name=*/tan_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, tan_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(\n kComplexHalf, common_dtype, \"tan_name\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n return ::tan(static_cast<opmath_t>(a));\n });\n });\n#endif\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::Half,\n ScalarType::BFloat16,\n common_dtype,\n \"tan_hip\",\n [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a) -> scalar_t { return ::tan(a); });\n });\n }\n}\n\nREGISTER_DISPATCH(tan_stub, &tan_kernel_hip);\n\n} // namespace at::native\n###"105 },106 {107 "cuda": "\n#include <ATen/core/Tensor.h>\n\nnamespace at {\nnamespace native {\nnamespace internal {\n\ntemplate <typename scalar_t>\nstd::tuple<Tensor, Tensor, Tensor> unique_cuda_template(\n const Tensor& self,\n const bool consecutive,\n const bool return_inverse,\n const bool return_counts);\n\n} // namespace internal\n} // namespace at\n} // namespace native\n\n\n###",108 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/core/Tensor.h>\n\nnamespace at {\nnamespace native {\nnamespace internal {\n\ntemplate <typename scalar_t>\nstd::tuple<Tensor, Tensor, Tensor> unique_hip_template(\n const Tensor& self,\n const bool consecutive,\n const bool return_inverse,\n const bool return_counts);\n\n} // namespace internal\n} // namespace at\n} // namespace native\n###"109 },110 {111 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/native/sparse/ValidateCompressedIndicesCommon.h>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\n\nnamespace {\n\ntemplate <typename func_t>\nstruct CUDAKernelLauncher {\n static void launch(TensorIteratorBase& iter, const func_t& f) {\n gpu_kernel(iter, f);\n }\n};\n\n}\n\nvoid _validate_compressed_sparse_indices_cuda(\n const bool is_crow,\n const Tensor& cidx,\n const Tensor& idx,\n const int64_t cdim,\n const int64_t dim,\n const int64_t nnz) {\n validate_compressed_sparse_indices_kernel<CUDAKernelLauncher>(\n is_crow, cidx, idx, cdim, dim, nnz);\n}\n\n} // namespace at::native\n\n\n###",112 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/native/sparse/ValidateCompressedIndicesCommon.h>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\n\nnamespace {\n\ntemplate <typename func_t>\nstruct HIPKernelLauncher {\n static void launch(TensorIteratorBase& iter, const func_t& f) {\n gpu_kernel(iter, f);\n }\n};\n\n}\n\nvoid _validate_compressed_sparse_indices_hip(\n const bool is_crow,\n const Tensor& cidx,\n const Tensor& idx,\n const int64_t cdim,\n const int64_t dim,\n const int64_t nnz) {\n validate_compressed_sparse_indices_kernel<HIPKernelLauncher>(\n is_crow, cidx, idx, cdim, dim, nnz);\n}\n\n} // namespace at::native\n###"113 },114 {115 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\nnamespace {\n\n/*\n * This function is derived from the implementation of the zeta function in the Cephes Math Library.\n * See note [3-Clause BSD License for the Cephes Math Library].\n */\n// See note [Jiterator]\nCONSTEXPR_EXCEPT_WIN_CUDA char zeta_name[] = \"zeta\";\nvoid zeta_kernel_cuda(TensorIteratorBase& iter) {\n #if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iter.common_dtype(), \"zeta_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars</*name=*/zeta_name,\n /*return_dtype=*/ scalar_t,\n /*f_inputs_dtype=*/ scalar_t>(iter, zeta_string);\n });\n #else\n AT_DISPATCH_FLOATING_TYPES(iter.common_dtype(), \"zeta_cuda\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t x, scalar_t q) -> scalar_t {\n return zeta<scalar_t, /*is_cuda=*/true>(x, q);\n });\n });\n #endif //jiterator\n}\n\n} // namespace (anonymous)\n\nREGISTER_DISPATCH(zeta_stub, &zeta_kernel_cuda);\n\n} // namespace at::native\n\n\n###",116 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\nnamespace {\n\n/*\n * This function is derived from the implementation of the zeta function in the Cephes Math Library.\n * See note [3-Clause BSD License for the Cephes Math Library].\n */\n// See note [Jiterator]\nCONSTEXPR_EXCEPT_WIN_HIP char zeta_name[] = \"zeta\";\nvoid zeta_kernel_hip(TensorIteratorBase& iter) {\n #if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iter.common_dtype(), \"zeta_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars</*name=*/zeta_name,\n /*return_dtype=*/ scalar_t,\n /*f_inputs_dtype=*/ scalar_t>(iter, zeta_string);\n });\n #else\n AT_DISPATCH_FLOATING_TYPES(iter.common_dtype(), \"zeta_hip\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t x, scalar_t q) -> scalar_t {\n return zeta<scalar_t, /*is_hip=*/true>(x, q);\n });\n });\n #endif //jiterator\n}\n\n} // namespace (anonymous)\n\nREGISTER_DISPATCH(zeta_stub, &zeta_kernel_hip);\n\n} // namespace at::native\n###"117 },118 {119 "cuda": "\n#include <ATen/ATen.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at {\nnamespace native {\n\nTensor& relu_quantized_cuda_(Tensor& self) {\n const auto zero_point = self.q_zero_point();\n AT_DISPATCH_QINT_TYPES(\n self.scalar_type(), \"qrelu_cuda\", [&]() {\n auto iter = TensorIterator::unary_op(self, self);\n gpu_kernel(iter, [zero_point] GPU_LAMBDA(scalar_t value) -> scalar_t {\n return scalar_t(std::max<underlying_t>(value.val_, zero_point));\n });\n });\n return self;\n}\n\n} // namespace at::native\n} // namespace at\n\n\n###",120 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/ATen.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at {\nnamespace native {\n\nTensor& relu_quantized_hip_(Tensor& self) {\n const auto zero_point = self.q_zero_point();\n AT_DISPATCH_QINT_TYPES(\n self.scalar_type(), \"qrelu_hip\", [&]() {\n auto iter = TensorIterator::unary_op(self, self);\n gpu_kernel(iter, [zero_point] GPU_LAMBDA(scalar_t value) -> scalar_t {\n return scalar_t(std::max<underlying_t>(value.val_, zero_point));\n });\n });\n return self;\n}\n\n} // namespace at::native\n} // namespace at\n###"121 },122 {123 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n#include <ATen/TensorIterator.h>\n#include <ATen/native/cuda/Loops.cuh>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/Functions.h>\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/empty.h>\n#include <ATen/ops/int_repr_native.h>\n#endif\n\nnamespace at {\nnamespace native {\n\nTensor int_repr_quantized_cuda(const Tensor& self) {\n Tensor dst;\n AT_DISPATCH_QINT_TYPES(self.scalar_type(), \"int_repr_quantized_cuda\", [&]() {\n dst = at::empty(\n self.sizes(),\n self.options().dtype(UNDERLYING_TYPE),\n self.suggest_memory_format());\n auto iter = TensorIteratorConfig()\n .check_all_same_dtype(false)\n .add_output(dst)\n .add_input(self)\n .build();\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t value) -> underlying_t {\n return value.val_;\n });\n });\n return dst;\n}\n\n} // namespace native\n} // namespace at\n\n\n###",124 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n#include <ATen/TensorIterator.h>\n#include <ATen/native/hip\\Loops.cuh>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/Functions.h>\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/empty.h>\n#include <ATen/ops/int_repr_native.h>\n#endif\n\nnamespace at {\nnamespace native {\n\nTensor int_repr_quantized_hip(const Tensor& self) {\n Tensor dst;\n AT_DISPATCH_QINT_TYPES(self.scalar_type(), \"int_repr_quantized_hip\", [&]() {\n dst = at::empty(\n self.sizes(),\n self.options().dtype(UNDERLYING_TYPE),\n self.suggest_memory_format());\n auto iter = TensorIteratorConfig()\n .check_all_same_dtype(false)\n .add_output(dst)\n .add_input(self)\n .build();\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t value) -> underlying_t {\n return value.val_;\n });\n });\n return dst;\n}\n\n} // namespace native\n} // namespace at\n###"125 },126 {127 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n#include <ATen/TensorIterator.h>\n#include <ATen/native/cuda/Loops.cuh>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/Functions.h>\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/_empty_affine_quantized.h>\n#include <ATen/ops/_empty_per_channel_affine_quantized.h>\n#include <ATen/ops/_make_per_channel_quantized_tensor_native.h>\n#include <ATen/ops/_make_per_tensor_quantized_tensor_native.h>\n#include <ATen/ops/empty.h>\n#endif\n\nnamespace at {\nnamespace native {\n\nvoid assign_quantized_tensor_cuda(\n const Tensor& self, Tensor& dst) {\n AT_DISPATCH_QINT_TYPES(\n dst.scalar_type(), \"assign_quantized_tensor_cuda\", [&]() {\n auto iter = TensorIteratorConfig()\n .check_all_same_dtype(false)\n .add_output(dst)\n .add_input(self)\n .build();\n gpu_kernel(iter, [] GPU_LAMBDA(underlying_t value) -> scalar_t {\n return scalar_t(value);\n });\n });\n}\n\nTensor make_per_tensor_quantized_tensor_cuda(\n const Tensor& self,\n double scale,\n int64_t zero_point) {\n Tensor dst = at::_empty_affine_quantized(\n self.sizes(),\n self.options().dtype(toQIntType(self.scalar_type())),\n scale,\n zero_point);\n assign_quantized_tensor_cuda(self, dst);\n return dst;\n}\n\nTensor make_per_channel_quantized_tensor_cuda(\n const Tensor& self,\n const Tensor& scales,\n const Tensor& zero_points,\n int64_t axis) {\n Tensor dst = at::_empty_per_channel_affine_quantized(\n self.sizes(),\n scales,\n zero_points,\n axis,\n self.options().dtype(toQIntType(self.scalar_type())));\n assign_quantized_tensor_cuda(self, dst);\n return dst;\n}\n\n} // namespace native\n} // namespace at\n\n\n###",128 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n#include <ATen/TensorIterator.h>\n#include <ATen/native/hip\\Loops.cuh>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/Functions.h>\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/_empty_affine_quantized.h>\n#include <ATen/ops/_empty_per_channel_affine_quantized.h>\n#include <ATen/ops/_make_per_channel_quantized_tensor_native.h>\n#include <ATen/ops/_make_per_tensor_quantized_tensor_native.h>\n#include <ATen/ops/empty.h>\n#endif\n\nnamespace at {\nnamespace native {\n\nvoid assign_quantized_tensor_hip(\n const Tensor& self, Tensor& dst) {\n AT_DISPATCH_QINT_TYPES(\n dst.scalar_type(), \"assign_quantized_tensor_hip\", [&]() {\n auto iter = TensorIteratorConfig()\n .check_all_same_dtype(false)\n .add_output(dst)\n .add_input(self)\n .build();\n gpu_kernel(iter, [] GPU_LAMBDA(underlying_t value) -> scalar_t {\n return scalar_t(value);\n });\n });\n}\n\nTensor make_per_tensor_quantized_tensor_hip(\n const Tensor& self,\n double scale,\n int64_t zero_point) {\n Tensor dst = at::_empty_affine_quantized(\n self.sizes(),\n self.options().dtype(toQIntType(self.scalar_type())),\n scale,\n zero_point);\n assign_quantized_tensor_hip(self, dst);\n return dst;\n}\n\nTensor make_per_channel_quantized_tensor_hip(\n const Tensor& self,\n const Tensor& scales,\n const Tensor& zero_points,\n int64_t axis) {\n Tensor dst = at::_empty_per_channel_affine_quantized(\n self.sizes(),\n scales,\n zero_points,\n axis,\n self.options().dtype(toQIntType(self.scalar_type())));\n assign_quantized_tensor_hip(self, dst);\n return dst;\n}\n\n} // namespace native\n} // namespace at\n###"129 },130 {131 "cuda": "\n#pragma once\n\n#include <ATen/cuda/detail/TensorInfo.cuh>\n#include <c10/macros/Macros.h>\n\nnamespace at {\nclass Tensor;\n}\nnamespace c10 {\nclass Scalar;\n}\n\nnamespace at { namespace native {\n\nvoid s_addmm_out_sparse_dense_cuda_worker(int64_t nnz, int64_t m, int64_t n, int64_t k, Tensor& r_, const Scalar& beta, const Tensor& t, const Scalar& alpha, Tensor& indices, Tensor& values, const Tensor& dense);\n\n}} // namespace at::native\n\n\n###",132 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <ATen/hip/detail\\TensorInfo.cuh>\n#include <c10/macros/Macros.h>\n\nnamespace at {\nclass Tensor;\n}\nnamespace c10 {\nclass Scalar;\n}\n\nnamespace at { namespace native {\n\nvoid s_addmm_out_sparse_dense_hip_worker(int64_t nnz, int64_t m, int64_t n, int64_t k, Tensor& r_, const Scalar& beta, const Tensor& t, const Scalar& alpha, Tensor& indices, Tensor& values, const Tensor& dense);\n\n}} // namespace at::native\n###"133 },134 {135 "cuda": "\n// No \"#pragma once\" because this is a raw definition that can be copied by jit codegen.\n// Eager mode clients should not include this file directly, instead,\n// they should #include <ATen/cuda/CUDAGeneratorImpl.h>, which has a #pragma once.\n\n// Stores RNG state values. Passed as a kernel argument.\n// See Note [CUDA Graph-safe RNG states].\n//\n// The raw definition lives in its own file so jit codegen can easily copy it.\nnamespace at {\n\nstruct PhiloxCudaState {\n PhiloxCudaState() = default;\n // Called if graph capture is not underway\n PhiloxCudaState(uint64_t seed,\n uint64_t offset) {\n seed_.val = seed;\n offset_.val = offset;\n }\n // Called if graph capture is underway\n PhiloxCudaState(int64_t* seed,\n int64_t* offset_extragraph,\n uint32_t offset_intragraph) {\n seed_.ptr = seed;\n offset_.ptr = offset_extragraph;\n offset_intragraph_ = offset_intragraph;\n captured_ = true;\n }\n\n // Public members, directly accessible by at::cuda::philox::unpack.\n // If we made them private with getters/setters, the getters/setters\n // would have to be __device__, and we can't declare __device__ in ATen.\n union Payload {\n uint64_t val;\n int64_t* ptr;\n };\n\n Payload seed_;\n Payload offset_;\n uint32_t offset_intragraph_ = 0;\n bool captured_ = false;\n};\n\n} // namespace at\n\n\n###",136 "hip": " // !!! This is a file automatically generated by hipify!!!\n// No \"#pragma once\" because this is a raw definition that can be copied by jit codegen.\n// Eager mode clients should not include this file directly, instead,\n// they should #include <ATen/hip\\HIPGeneratorImpl.h>, which has a #pragma once.\n\n// Stores RNG state values. Passed as a kernel argument.\n// See Note [HIP Graph-safe RNG states].\n//\n// The raw definition lives in its own file so jit codegen can easily copy it.\nnamespace at {\n\nstruct PhiloxHipState {\n PhiloxHipState() = default;\n // Called if graph capture is not underway\n PhiloxHipState(uint64_t seed,\n uint64_t offset) {\n seed_.val = seed;\n offset_.val = offset;\n }\n // Called if graph capture is underway\n PhiloxHipState(int64_t* seed,\n int64_t* offset_extragraph,\n uint32_t offset_intragraph) {\n seed_.ptr = seed;\n offset_.ptr = offset_extragraph;\n offset_intragraph_ = offset_intragraph;\n captured_ = true;\n }\n\n // Public members, directly accessible by at::cuda::philox::unpack.\n // If we made them private with getters/setters, the getters/setters\n // would have to be __device__, and we can't declare __device__ in ATen.\n union Payload {\n uint64_t val;\n int64_t* ptr;\n };\n\n Payload seed_;\n Payload offset_;\n uint32_t offset_intragraph_ = 0;\n bool captured_ = false;\n};\n\n} // namespace at\n###"137 },138 {139 "cuda": "\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/cuda/flash_attn/fmha_bwd_launch_template.h>\n\nvoid run_fmha_bwd_hdim128(FMHA_dgrad_params ¶ms, cudaStream_t stream, const bool configure) {\n FP16_SWITCH(params.is_bf16, ([&] {\n using Kernel_traits = FMHA_kernel_traits<128, 128, 16, 1, 8, 0x100u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n }));\n}\n\n###",140 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/hip/flash_attn\\fmha_bwd_launch_template.h>\n\nvoid run_fmha_bwd_hdim128(FMHA_dgrad_params ¶ms, hipStream_t stream, const bool configure) {\n FP16_SWITCH(params.is_bf16, ([&] {\n using Kernel_traits = FMHA_kernel_traits<128, 128, 16, 1, 8, 0x100u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n }));\n}###"141 },142 {143 "cuda": "\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/cuda/flash_attn/fmha_bwd_launch_template.h>\n\nvoid run_fmha_bwd_hdim32(FMHA_dgrad_params ¶ms, cudaStream_t stream, const bool configure) {\n FP16_SWITCH(params.is_bf16, ([&] {\n if (params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 32, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (params.seqlen_k >= 256) {\n using Kernel_traits = FMHA_kernel_traits<256, 32, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n }\n }));\n}\n\n###",144 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/hip/flash_attn\\fmha_bwd_launch_template.h>\n\nvoid run_fmha_bwd_hdim32(FMHA_dgrad_params ¶ms, hipStream_t stream, const bool configure) {\n FP16_SWITCH(params.is_bf16, ([&] {\n if (params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 32, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (params.seqlen_k >= 256) {\n using Kernel_traits = FMHA_kernel_traits<256, 32, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n }\n }));\n}###"145 },146 {147 "cuda": "\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/cuda/flash_attn/fmha_bwd_launch_template.h>\n\nvoid run_fmha_bwd_hdim64(FMHA_dgrad_params ¶ms, cudaStream_t stream, const bool configure) {\n FP16_SWITCH(params.is_bf16, ([&] {\n auto dprops = at::cuda::getCurrentDeviceProperties();\n if (params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 64, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (params.seqlen_k >= 256) {\n if ((dprops->major == 8 && dprops->minor == 0) ||\n (dprops->major == 9 && dprops->minor == 0)) {\n // Don't share smem for K & V, and don't keep V in registers\n // This speeds things up by 2-3% by avoiding register spills, but it\n // uses more shared memory, which is fine on A100 and H100 but not other\n // GPUs. For other GPUs, we keep V in registers.\n using Kernel_traits =\n FMHA_kernel_traits<256, 64, 16, 1, 8, 0x100u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (dprops->major == 8 && dprops->minor > 0) {\n using Kernel_traits =\n FMHA_kernel_traits<256, 64, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (dprops->major == 7 && dprops->minor == 5) {\n using Kernel_traits =\n FMHA_kernel_traits<128, 64, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n }\n }\n }));\n}\n\n###",148 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/hip/flash_attn\\fmha_bwd_launch_template.h>\n\nvoid run_fmha_bwd_hdim64(FMHA_dgrad_params ¶ms, hipStream_t stream, const bool configure) {\n FP16_SWITCH(params.is_bf16, ([&] {\n auto dprops = at::cuda::getCurrentDeviceProperties();\n if (params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 64, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (params.seqlen_k >= 256) {\n if ((dprops->major == 8 && dprops->minor == 0) ||\n (dprops->major == 9 && dprops->minor == 0)) {\n // Don't share smem for K & V, and don't keep V in registers\n // This speeds things up by 2-3% by avoiding register spills, but it\n // uses more shared memory, which is fine on A100 and H100 but not other\n // GPUs. For other GPUs, we keep V in registers.\n using Kernel_traits =\n FMHA_kernel_traits<256, 64, 16, 1, 8, 0x100u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (dprops->major == 8 && dprops->minor > 0) {\n using Kernel_traits =\n FMHA_kernel_traits<256, 64, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n } else if (dprops->major == 7 && dprops->minor == 5) {\n using Kernel_traits =\n FMHA_kernel_traits<128, 64, 16, 1, 8, 0x08u, elem_type>;\n run_fmha_bwd_loop<Kernel_traits>(params, stream, configure);\n }\n }\n }));\n}###"149 },150 {151 "cuda": "\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/cuda/flash_attn/fmha_fwd_launch_template.h>\n\nvoid run_fmha_fwd_hdim128(Launch_params<FMHA_fprop_params> &launch_params) {\n FP16_SWITCH(launch_params.params.is_bf16, ([&] {\n using Kernel_traits = FMHA_kernel_traits<128, 128, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n }));\n}\n\n###",152 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/hip/flash_attn\\fmha_fwd_launch_template.h>\n\nvoid run_fmha_fwd_hdim128(Launch_params<FMHA_fprop_params> &launch_params) {\n FP16_SWITCH(launch_params.params.is_bf16, ([&] {\n using Kernel_traits = FMHA_kernel_traits<128, 128, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n }));\n}###"153 },154 {155 "cuda": "\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/cuda/flash_attn/fmha_fwd_launch_template.h>\n\nvoid run_fmha_fwd_hdim32(Launch_params<FMHA_fprop_params> &launch_params) {\n FP16_SWITCH(launch_params.params.is_bf16, ([&] {\n if (launch_params.params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 32, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n } else if (launch_params.params.seqlen_k >= 256) {\n using Kernel_traits = FMHA_kernel_traits<256, 32, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n }\n }));\n}\n\n###",156 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/hip/flash_attn\\fmha_fwd_launch_template.h>\n\nvoid run_fmha_fwd_hdim32(Launch_params<FMHA_fprop_params> &launch_params) {\n FP16_SWITCH(launch_params.params.is_bf16, ([&] {\n if (launch_params.params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 32, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n } else if (launch_params.params.seqlen_k >= 256) {\n using Kernel_traits = FMHA_kernel_traits<256, 32, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n }\n }));\n}###"157 },158 {159 "cuda": "\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/cuda/flash_attn/fmha_fwd_launch_template.h>\n\nvoid run_fmha_fwd_hdim64(Launch_params<FMHA_fprop_params> &launch_params) {\n FP16_SWITCH(launch_params.params.is_bf16, ([&] {\n if (launch_params.params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 64, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n } else if (launch_params.params.seqlen_k >= 256) {\n using Kernel_traits = FMHA_kernel_traits<256, 64, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n }\n }));\n}\n\n\n###",160 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) 2022, Tri Dao.\n\n// Splitting the different head dimensions to different files to speed up compilation.\n\n#include <ATen/native/transformers/hip/flash_attn\\fmha_fwd_launch_template.h>\n\nvoid run_fmha_fwd_hdim64(Launch_params<FMHA_fprop_params> &launch_params) {\n FP16_SWITCH(launch_params.params.is_bf16, ([&] {\n if (launch_params.params.seqlen_k == 128) {\n using Kernel_traits = FMHA_kernel_traits<128, 64, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n } else if (launch_params.params.seqlen_k >= 256) {\n using Kernel_traits = FMHA_kernel_traits<256, 64, 16, 1, 4, 0x08u, elem_type>;\n run_fmha_fwd_loop<Kernel_traits>(launch_params);\n }\n }));\n}\n###"161 },162 {163 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x128_k128_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x128_k128_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k128_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k128_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",164 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x128_k128_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 128, 128, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x128_k128_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k128_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k128_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"165 },166 {167 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k32_seqaligned_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k32_seqaligned_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k32_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k32_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",168 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k32_seqaligned_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32, true>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k32_seqaligned_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k32_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k32_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"169 },170 {171 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k32_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k32_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",172 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k32_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k32_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"173 },174 {175 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k64_seqaligned_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k64_seqaligned_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k64_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k64_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",176 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k64_seqaligned_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64, true>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k64_seqaligned_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k64_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k64_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"177 },178 {179 "cuda": "\n#pragma once\n#include <ATen/CollapseDims.h>\nnamespace at {\nnamespace cuda {\nnamespace detail {\n#define MAX_TENSORINFO_DIMS 25\n\ntemplate <typename T, typename IndexType>\nstruct TensorInfo {\n TensorInfo();\n TensorInfo(T* p, int dim, IndexType sz[MAX_TENSORINFO_DIMS], IndexType st[MAX_TENSORINFO_DIMS]);\n \n \n \n void reduceDim(int dim);\n \n int collapseDims(const int excludeDim = -1);\n \n \n __host__ __device__ inline bool isContiguous() const {\n return (dims == 1 && strides[0] == 1);\n }\n T* data;\n IndexType sizes[MAX_TENSORINFO_DIMS];\n IndexType strides[MAX_TENSORINFO_DIMS];\n int dims;\n};\ntemplate <typename T, typename IndexType>\nTensorInfo<T, IndexType>::TensorInfo() {\n data = nullptr;\n dims = 0;\n}\ntemplate <typename T, typename IndexType>\nTensorInfo<T, IndexType>::TensorInfo(T* p, int dim, IndexType sz[MAX_TENSORINFO_DIMS], IndexType st[MAX_TENSORINFO_DIMS]) {\n data = p;\n dims = dim;\n TORCH_CHECK(dims < MAX_TENSORINFO_DIMS, \"CUDA Tensors cannot have more than 25 dimensions\");\n for (int i = 0; i < dim; ++i) {\n sizes[i] = sz[i];\n strides[i] = st[i];\n }\n}\ntemplate <typename T, typename IndexType>\nvoid\nTensorInfo<T, IndexType>::reduceDim(int dim) {\n TORCH_CHECK(dim < dims && dim >= 0, \"expected dim between 0 and dims - 1\");\n sizes[dim] = 1;\n}\ntemplate <typename T, typename IndexType>\nint\nTensorInfo<T, IndexType>::collapseDims(const int excludeDim) {\n auto result = at::collapse_dims(sizes, strides, dims, excludeDim);\n dims = std::get<1>(result);\n return std::get<0>(result);\n}\n\n\ntemplate <typename T, typename IndexType, int Dims>\nstruct IndexToOffset {\n static __host__ __device__ IndexType get(\n IndexType linearId, const TensorInfo<T, IndexType>& info) {\n IndexType offset = 0;\n \n for (int i = Dims - 1; i > 0; --i) {\n IndexType curDimIndex = linearId % info.sizes[i];\n IndexType curDimOffset = curDimIndex * info.strides[i];\n offset += curDimOffset;\n linearId /= info.sizes[i];\n }\n return offset + linearId * info.strides[0];\n }\n};\n\ntemplate <typename T, typename IndexType>\nstruct IndexToOffset<T, IndexType, -1> {\n static inline __host__ __device__ IndexType get(\n IndexType linearId, const TensorInfo<T, IndexType>& info) {\n IndexType offset = 0;\n for (int i = info.dims - 1; i > 0; --i) {\n IndexType curDimIndex = linearId % info.sizes[i];\n IndexType curDimOffset = curDimIndex * info.strides[i];\n offset += curDimOffset;\n linearId /= info.sizes[i];\n }\n return offset + linearId * info.strides[0];\n }\n};\n} \n} \n} \n\n###",180 "hip": " \n#pragma once\n#include <ATen/CollapseDims.h>\nnamespace at {\nnamespace hip {\nnamespace detail {\n#define MAX_TENSORINFO_DIMS 25\n\ntemplate <typename T, typename IndexType>\nstruct TensorInfo {\n TensorInfo();\n TensorInfo(T* p, int dim, IndexType sz[MAX_TENSORINFO_DIMS], IndexType st[MAX_TENSORINFO_DIMS]);\n \n \n \n void reduceDim(int dim);\n \n int collapseDims(const int excludeDim = -1);\n \n \n __host__ __device__ inline bool isContiguous() const {\n return (dims == 1 && strides[0] == 1);\n }\n T* data;\n IndexType sizes[MAX_TENSORINFO_DIMS];\n IndexType strides[MAX_TENSORINFO_DIMS];\n int dims;\n};\ntemplate <typename T, typename IndexType>\nTensorInfo<T, IndexType>::TensorInfo() {\n data = nullptr;\n dims = 0;\n}\ntemplate <typename T, typename IndexType>\nTensorInfo<T, IndexType>::TensorInfo(T* p, int dim, IndexType sz[MAX_TENSORINFO_DIMS], IndexType st[MAX_TENSORINFO_DIMS]) {\n data = p;\n dims = dim;\n TORCH_CHECK(dims < MAX_TENSORINFO_DIMS, \"HIP Tensors cannot have more than 25 dimensions\");\n for (int i = 0; i < dim; ++i) {\n sizes[i] = sz[i];\n strides[i] = st[i];\n }\n}\ntemplate <typename T, typename IndexType>\nvoid\nTensorInfo<T, IndexType>::reduceDim(int dim) {\n TORCH_CHECK(dim < dims && dim >= 0, \"expected dim between 0 and dims - 1\");\n sizes[dim] = 1;\n}\ntemplate <typename T, typename IndexType>\nint\nTensorInfo<T, IndexType>::collapseDims(const int excludeDim) {\n auto result = at::collapse_dims(sizes, strides, dims, excludeDim);\n dims = std::get<1>(result);\n return std::get<0>(result);\n}\n\n\ntemplate <typename T, typename IndexType, int Dims>\nstruct IndexToOffset {\n static __host__ __device__ IndexType get(\n IndexType linearId, const TensorInfo<T, IndexType>& info) {\n IndexType offset = 0;\n \n for (int i = Dims - 1; i > 0; --i) {\n IndexType curDimIndex = linearId % info.sizes[i];\n IndexType curDimOffset = curDimIndex * info.strides[i];\n offset += curDimOffset;\n linearId /= info.sizes[i];\n }\n return offset + linearId * info.strides[0];\n }\n};\n\ntemplate <typename T, typename IndexType>\nstruct IndexToOffset<T, IndexType, -1> {\n static inline __host__ __device__ IndexType get(\n IndexType linearId, const TensorInfo<T, IndexType>& info) {\n IndexType offset = 0;\n for (int i = info.dims - 1; i > 0; --i) {\n IndexType curDimIndex = linearId % info.sizes[i];\n IndexType curDimOffset = curDimIndex * info.strides[i];\n offset += curDimOffset;\n linearId /= info.sizes[i];\n }\n return offset + linearId * info.strides[0];\n }\n};\n} \n} \n} ###"181 },182 {183 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k64_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k64_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",184 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k64_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, true, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k64_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"185 },186 {187 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x64_k65536_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x64_k65536_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k65536_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k65536_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",188 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x64_k65536_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 128, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x64_k65536_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k65536_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k65536_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"189 },190 {191 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x64_k65536_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x64_k65536_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k65536_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k65536_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",192 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x64_k65536_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 128, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x64_k65536_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_64x64_k65536_dropout_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, true, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_64x64_k65536_dropout_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"193 },194 {195 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x64_k96_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x64_k96_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",196 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::kMinBlocksPerSm)\nfmha_cutlassB_bf16_aligned_128x64_k96_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::bfloat16_t, true, false, true, 128, 64, 96>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_bf16_aligned_128x64_k96_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"197 },198 {199 "cuda": "\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::kMinBlocksPerSm)\nfmha_cutlassB_f16_aligned_128x64_k96_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 800\n#if __CUDA_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f16_aligned_128x64_k96_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n\n###",200 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/*\n * Copyright (c) Meta Platforms, Inc. and affiliates.\n * All rights reserved.\n *\n * This source code is licensed under the BSD-style license found in the\n * LICENSE file in the root directory of this source tree.\n */\n// This file is auto-generated. See \"generate_kernels.py\"\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::kNumThreads,\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::kMinBlocksPerSm)\nfmha_cutlassB_f16_aligned_128x64_k96_sm80(typename AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 800\n#if __HIP_ARCH__ < 1000\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm80, cutlass::half_t, true, false, true, 128, 64, 96>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f16_aligned_128x64_k96_sm80` is for sm80-sm100, but was built for sm%d\\n\",\n int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n###"201 },202 {203 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",204 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"205 },206 {207 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",208 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 128>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k128_dropout_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"209 },210 {211 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",212 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"213 },214 {215 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",216 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 32>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k32_dropout_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"217 },218 {219 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",220 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"221 },222 {223 "cuda": "\n// No \"#pragma once\" because this is a raw definition that can be copied by jit codegen.\n// Eager mode clients should not include this file directly, instead,\n// they should #include <ATen/cuda/CUDAGraphsUtils.cuh>, which has a #pragma once.\n\nnamespace at {\nnamespace cuda {\nnamespace philox {\n\n// In-kernel call to retrieve philox seed and offset from a PhiloxCudaState instance whether\n// that instance was created with graph capture underway or not.\n// See Note [CUDA Graph-safe RNG states].\n//\n// We can't write a __device__ function in CUDAGeneratorImpl.h, because it's in ATen.\n// Also, whatever call unpacks PhiloxCudaState in consumer kernels must be inlineable.\n// Easiest thing that comes to mind is, define a __device__ unpack helper here, in ATen/cuda.\n//\n// The raw definition lives in its own file so jit codegen can easily copy it.\n__device__ __forceinline__ std::tuple<uint64_t, uint64_t>\nunpack(at::PhiloxCudaState arg) {\n if (arg.captured_) {\n // static_cast avoids \"warning: invalid narrowing conversion from \"long\" to \"unsigned long\".\n // *(arg.offset_.ptr) is a broadcast load of a single int64_t to the entire kernel.\n // For most threads' reads it will hit in cache, so it shouldn't hurt performance.\n return std::make_tuple(static_cast<uint64_t>(*arg.seed_.ptr), static_cast<uint64_t>(*(arg.offset_.ptr) + arg.offset_intragraph_));\n } else {\n return std::make_tuple(arg.seed_.val, arg.offset_.val);\n }\n}\n\n} // namespace philox\n} // namespace cuda\n} // namespace at\n\n\n###",224 "hip": " // !!! This is a file automatically generated by hipify!!!\n// No \"#pragma once\" because this is a raw definition that can be copied by jit codegen.\n// Eager mode clients should not include this file directly, instead,\n// they should #include <ATen/hip\\HIPGraphsUtils.cuh>, which has a #pragma once.\n\nnamespace at {\nnamespace hip {\nnamespace philox {\n\n// In-kernel call to retrieve philox seed and offset from a PhiloxHipState instance whether\n// that instance was created with graph capture underway or not.\n// See Note [HIP Graph-safe RNG states].\n//\n// We can't write a __device__ function in HIPGeneratorImpl.h, because it's in ATen.\n// Also, whatever call unpacks PhiloxHipState in consumer kernels must be inlineable.\n// Easiest thing that comes to mind is, define a __device__ unpack helper here, in ATen/cuda.\n//\n// The raw definition lives in its own file so jit codegen can easily copy it.\n__device__ __forceinline__ std::tuple<uint64_t, uint64_t>\nunpack(at::PhiloxHipState arg) {\n if (arg.captured_) {\n // static_cast avoids \"warning: invalid narrowing conversion from \"long\" to \"unsigned long\".\n // *(arg.offset_.ptr) is a broadcast load of a single int64_t to the entire kernel.\n // For most threads' reads it will hit in cache, so it shouldn't hurt performance.\n return std::make_tuple(static_cast<uint64_t>(*arg.seed_.ptr), static_cast<uint64_t>(*(arg.offset_.ptr) + arg.offset_intragraph_));\n } else {\n return std::make_tuple(arg.seed_.val, arg.offset_.val);\n }\n}\n\n} // namespace philox\n} // namespace hip\n} // namespace at\n###"225 },226 {227 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",228 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, true, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, true, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, true, false, 64, 64, 64>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k64_dropout_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"229 },230 {231 "cuda": "\n\n\n#include <ATen/native/transformers/cuda/mem_eff_attention/kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k65536_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 500\n#if __CUDA_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k65536_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k65536_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 700\n#if __CUDA_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k65536_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k65536_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::Params p) {\n#ifdef __CUDA_ARCH__\n#if __CUDA_ARCH__ >= 750\n#if __CUDA_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k65536_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__CUDA_ARCH__ + 0) / 10);\n#endif\n}\n\n###",232 "hip": " \n#include \"hip/hip_runtime.h\"\n\n\n#include <ATen/native/transformers/hip/mem_eff_attention\\kernel_backward.h>\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k65536_sm50(typename AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 500\n#if __HIP_ARCH__ < 700\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm50, float, false, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k65536_sm50` is for sm50-sm70, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k65536_sm70(typename AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 700\n#if __HIP_ARCH__ < 750\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm70, float, false, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k65536_sm70` is for sm70-sm75, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}\n__global__ void __launch_bounds__(\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::kNumThreads, AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::kMinBlocksPerSm)\nfmha_cutlassB_f32_notaligned_64x64_k65536_sm75(typename AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::Params p) {\n#ifdef __HIP_ARCH__\n#if __HIP_ARCH__ >= 750\n#if __HIP_ARCH__ < 800\n if (!p.advance_to_block()) {\n return;\n }\n AttentionBackwardKernel<cutlass::arch::Sm75, float, false, false, false, 64, 64, 65536>::attention_kernel(p);\n return;\n#endif\n#endif\n printf(\n \"FATAL: kernel `fmha_cutlassB_f32_notaligned_64x64_k65536_sm75` is for sm75-sm80, but was built for sm%d\\n\", int(__HIP_ARCH__ + 0) / 10);\n#endif\n}###"233 },234 {235 "cuda": "\n#include <gtest/gtest.h>\n#include <c10/cuda/CUDAException.h>\n\nint safeDeviceCount() {\n int count;\n cudaError_t err = cudaGetDeviceCount(&count);\n if (err == cudaErrorInsufficientDriver || err == cudaErrorNoDevice) {\n return 0;\n }\n return count;\n}\n\n#define SKIP_IF_NO_GPU() \\\n do { \\\n if (safeDeviceCount() == 0) { \\\n return; \\\n } \\\n } while(0)\n\n#define C10_ASSERT_NEAR(a, b, tol) assert(abs(a - b) < tol)\n#define C10_DEFINE_TEST(a, b) \\\n__global__ void CUDA##a##b(); \\\nTEST(a##Device, b) { \\\n SKIP_IF_NO_GPU(); \\\n cudaDeviceSynchronize(); \\\n CUDA##a##b<<<1, 1>>>(); \\\n C10_CUDA_KERNEL_LAUNCH_CHECK(); \\\n cudaDeviceSynchronize(); \\\n ASSERT_EQ(cudaGetLastError(), cudaSuccess); \\\n} \\\n__global__ void CUDA##a##b()\n#include <c10/test/util/complex_math_test_common.h>\n\n\n#undef C10_DEFINE_TEST\n#undef C10_ASSERT_NEAR\n#define C10_DEFINE_TEST(a, b) TEST(a##Host, b)\n#define C10_ASSERT_NEAR(a, b, tol) ASSERT_NEAR(a, b, tol)\n#include <c10/test/util/complex_math_test_common.h>\n\n\n###",236 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <gtest/gtest.h>\n#include <c10/hip/HIPException.h>\n\nint safeDeviceCount() {\n int count;\n hipError_t err = hipGetDeviceCount(&count);\n if (err == hipErrorInsufficientDriver || err == hipErrorNoDevice) {\n return 0;\n }\n return count;\n}\n\n#define SKIP_IF_NO_GPU() \\\n do { \\\n if (safeDeviceCount() == 0) { \\\n return; \\\n } \\\n } while(0)\n\n#define C10_ASSERT_NEAR(a, b, tol) assert(abs(a - b) < tol)\n#define C10_DEFINE_TEST(a, b) \\\n__global__ void HIP##a##b(); \\\nTEST(a##Device, b) { \\\n SKIP_IF_NO_GPU(); \\\n hipDeviceSynchronize(); \\\n hipLaunchKernelGGL(( HIP##a##b), dim3(1), dim3(1), 0, 0, ); \\\n C10_HIP_KERNEL_LAUNCH_CHECK(); \\\n hipDeviceSynchronize(); \\\n ASSERT_EQ(hipGetLastError(), hipSuccess); \\\n} \\\n__global__ void HIP##a##b()\n#include <c10/test/util/complex_math_test_common.h>\n\n\n#undef C10_DEFINE_TEST\n#undef C10_ASSERT_NEAR\n#define C10_DEFINE_TEST(a, b) TEST(a##Host, b)\n#define C10_ASSERT_NEAR(a, b, tol) ASSERT_NEAR(a, b, tol)\n#include <c10/test/util/complex_math_test_common.h>\n###"237 },238 {239 "cuda": "\n#include <gtest/gtest.h>\n\n#include <ATen/ATen.h>\n#include <ATen/cuda/CUDAContext.h>\n#include <c10/util/Optional.h>\n\n#include <assert.h>\n\nusing namespace at;\n\n// optional in cuda files\nTEST(OptionalTest, OptionalTestCUDA) {\n if (!at::cuda::is_available()) return;\n c10::optional<int64_t> trivially_destructible;\n c10::optional<std::vector<int64_t>> non_trivially_destructible;\n ASSERT_FALSE(trivially_destructible.has_value());\n ASSERT_FALSE(non_trivially_destructible.has_value());\n\n trivially_destructible = {5};\n non_trivially_destructible = std::vector<int64_t>{5, 10};\n ASSERT_TRUE(trivially_destructible.has_value());\n ASSERT_TRUE(non_trivially_destructible.has_value());\n}\n\n\n###",240 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <gtest/gtest.h>\n\n#include <ATen/ATen.h>\n#include <ATen/hip\\HIPContext.h>\n#include <c10/util/Optional.h>\n\n#include <assert.h>\n\nusing namespace at;\n\n// optional in cuda files\nTEST(OptionalTest, OptionalTestHIP) {\n if (!at::cuda::is_available()) return;\n c10::optional<int64_t> trivially_destructible;\n c10::optional<std::vector<int64_t>> non_trivially_destructible;\n ASSERT_FALSE(trivially_destructible.has_value());\n ASSERT_FALSE(non_trivially_destructible.has_value());\n\n trivially_destructible = {5};\n non_trivially_destructible = std::vector<int64_t>{5, 10};\n ASSERT_TRUE(trivially_destructible.has_value());\n ASSERT_TRUE(non_trivially_destructible.has_value());\n}\n###"241 },242 {243 "cuda": "\n#include <gtest/gtest.h>\n\n#include <ATen/ATen.h>\n#include <ATen/core/TensorAccessor.h>\n#include <ATen/cuda/CUDAContext.h>\n\n#include <assert.h>\n\nusing namespace at;\n\n__global__ void test_tensor_packed_accessor_kernel(\n PackedTensorAccessor64<float, 1, RestrictPtrTraits> resa,\n PackedTensorAccessor64<float, 2, RestrictPtrTraits> t1a,\n PackedTensorAccessor64<float, 1, RestrictPtrTraits> t2a) {\n for (int64_t i = 0; i < resa.size(0); i++) {\n float val = 0.0f;\n for (int64_t j = 0; j < t1a.size(1); j++) {\n val += t1a[i][j] * t2a[j];\n }\n resa[i] = val;\n }\n}\n\n// test GenericPackedTensorAccessor and Tensor.generic_packed_accessor\nTEST(PackedtensoraccessorTest, PackedtensoraccessorTestCUDA) {\n if (!at::cuda::is_available()) return;\n manual_seed(123);\n\n Tensor t1 = rand({4, 4}, CUDA(kFloat));\n Tensor t2 = rand({4}, CUDA(kFloat));\n Tensor res = empty({4}, CUDA(kFloat));\n\n auto t1a = t1.packed_accessor64<float, 2, RestrictPtrTraits>();\n auto t2a = t2.packed_accessor64<float, 1, RestrictPtrTraits>();\n auto resa = res.packed_accessor64<float, 1, RestrictPtrTraits>();\n\n auto stream = at::cuda::getCurrentCUDAStream();\n\n test_tensor_packed_accessor_kernel<<<1, 1, 0, stream>>>(resa, t1a, t2a);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n ASSERT_TRUE(cudaSuccess == cudaDeviceSynchronize());\n\n auto expected = mv(t1, t2);\n\n ASSERT_TRUE(res.allclose(expected));\n}\n\n\n###",244 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <gtest/gtest.h>\n\n#include <ATen/ATen.h>\n#include <ATen/core/TensorAccessor.h>\n#include <ATen/hip\\HIPContext.h>\n\n#include <assert.h>\n\nusing namespace at;\n\n__global__ void test_tensor_packed_accessor_kernel(\n PackedTensorAccessor64<float, 1, RestrictPtrTraits> resa,\n PackedTensorAccessor64<float, 2, RestrictPtrTraits> t1a,\n PackedTensorAccessor64<float, 1, RestrictPtrTraits> t2a) {\n for (int64_t i = 0; i < resa.size(0); i++) {\n float val = 0.0f;\n for (int64_t j = 0; j < t1a.size(1); j++) {\n val += t1a[i][j] * t2a[j];\n }\n resa[i] = val;\n }\n}\n\n// test GenericPackedTensorAccessor and Tensor.generic_packed_accessor\nTEST(PackedtensoraccessorTest, PackedtensoraccessorTestHIP) {\n if (!at::cuda::is_available()) return;\n manual_seed(123);\n\n Tensor t1 = rand({4, 4}, HIP(kFloat));\n Tensor t2 = rand({4}, HIP(kFloat));\n Tensor res = empty({4}, HIP(kFloat));\n\n auto t1a = t1.packed_accessor64<float, 2, RestrictPtrTraits>();\n auto t2a = t2.packed_accessor64<float, 1, RestrictPtrTraits>();\n auto resa = res.packed_accessor64<float, 1, RestrictPtrTraits>();\n\n auto stream = at::hip::getCurrentHIPStream();\n\n hipLaunchKernelGGL(( test_tensor_packed_accessor_kernel), dim3(1), dim3(1), 0, stream, resa, t1a, t2a);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n ASSERT_TRUE(hipSuccess == hipDeviceSynchronize());\n\n auto expected = mv(t1, t2);\n\n ASSERT_TRUE(res.allclose(expected));\n}\n###"245 },246 {247 "cuda": "\n#pragma once\n// TODO: Remove once torchvision has been updated to use the ATen header\n#include <ATen/cuda/Atomic.cuh>\n\n\n###",248 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n// TODO: Remove once torchvision has been updated to use the ATen header\n#include <ATen/hip\\Atomic.cuh>\n###"249 },250 {251 "cuda": "\n#pragma once\n// TODO: Remove this header\n#include <ATen/cuda/DeviceUtils.cuh>\n\n\n###",252 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n// TODO: Remove this header\n#include <ATen/hip\\DeviceUtils.cuh>\n###"253 },254 {255 "cuda": "\n/**\n * Copyright (c) 2016-present, Facebook, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <cuda_runtime.h>\n\n#include <sstream>\n#include <vector>\n\n#include \"c10/util/Flags.h\"\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/init.h\"\n#include \"caffe2/core/logging.h\"\n\nusing std::vector;\n\nC10_DECLARE_int(caffe2_log_level);\n\nint main(int argc, char** argv) {\n caffe2::GlobalInit(&argc, &argv);\n c10::SetUsageMessage(\n \"Inspects the GPUs on the current machine and prints out their details \"\n \"provided by cuda.\");\n\n int gpu_count;\n CUDA_ENFORCE(cudaGetDeviceCount(&gpu_count));\n for (int i = 0; i < gpu_count; ++i) {\n LOG(INFO) << \"Querying device ID = \" << i;\n caffe2::DeviceQuery(i);\n }\n\n vector<vector<bool> > access_pattern;\n CAFFE_ENFORCE(caffe2::GetCudaPeerAccessPattern(&access_pattern));\n\n std::stringstream sstream;\n // Find topology\n for (int i = 0; i < gpu_count; ++i) {\n for (int j = 0; j < gpu_count; ++j) {\n sstream << (access_pattern[i][j] ? \"+\" : \"-\") << \" \";\n }\n sstream << std::endl;\n }\n LOG(INFO) << \"Access pattern: \" << std::endl << sstream.str();\n\n return 0;\n}\n\n\n###",256 "hip": " // !!! This is a file automatically generated by hipify!!!\n/**\n * Copyright (c) 2016-present, Facebook, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <hip/hip_runtime.h>\n\n#include <sstream>\n#include <vector>\n\n#include \"c10/util/Flags.h\"\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/init.h\"\n#include \"caffe2/core/logging.h\"\n\nusing std::vector;\n\nC10_DECLARE_int(caffe2_log_level);\n\nint main(int argc, char** argv) {\n caffe2::GlobalInit(&argc, &argv);\n c10::SetUsageMessage(\n \"Inspects the GPUs on the current machine and prints out their details \"\n \"provided by cuda.\");\n\n int gpu_count;\n HIP_ENFORCE(hipGetDeviceCount(&gpu_count));\n for (int i = 0; i < gpu_count; ++i) {\n LOG(INFO) << \"Querying device ID = \" << i;\n caffe2::DeviceQuery(i);\n }\n\n vector<vector<bool> > access_pattern;\n CAFFE_ENFORCE(caffe2::GetHipPeerAccessPattern(&access_pattern));\n\n std::stringstream sstream;\n // Find topology\n for (int i = 0; i < gpu_count; ++i) {\n for (int j = 0; j < gpu_count; ++j) {\n sstream << (access_pattern[i][j] ? \"+\" : \"-\") << \" \";\n }\n sstream << std::endl;\n }\n LOG(INFO) << \"Access pattern: \" << std::endl << sstream.str();\n\n return 0;\n}\n###"257 },258 {259 "cuda": "\n/**\n * Copyright (c) 2016-present, Facebook, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <iostream>\n\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/proto/caffe2_pb.h\"\n\n#define PRINT_SIZE(cls) \\\n std::cout << \"Size of \" #cls \": \" << sizeof(cls) << \" bytes.\" \\\n << std::endl;\n\nint main(int /* unused */, char** /* unused */) {\n PRINT_SIZE(caffe2::Blob);\n PRINT_SIZE(caffe2::Tensor);\n PRINT_SIZE(caffe2::CPUContext);\n PRINT_SIZE(caffe2::CUDAContext);\n PRINT_SIZE(caffe2::OperatorBase);\n PRINT_SIZE(caffe2::OperatorDef);\n PRINT_SIZE(caffe2::Operator<caffe2::CPUContext>);\n PRINT_SIZE(caffe2::Operator<caffe2::CUDAContext>);\n PRINT_SIZE(caffe2::TypeMeta);\n PRINT_SIZE(caffe2::Workspace);\n return 0;\n}\n\n\n###",260 "hip": " // !!! This is a file automatically generated by hipify!!!\n/**\n * Copyright (c) 2016-present, Facebook, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include <iostream>\n\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/proto/caffe2_pb.h\"\n\n#define PRINT_SIZE(cls) \\\n std::cout << \"Size of \" #cls \": \" << sizeof(cls) << \" bytes.\" \\\n << std::endl;\n\nint main(int /* unused */, char** /* unused */) {\n PRINT_SIZE(caffe2::Blob);\n PRINT_SIZE(caffe2::Tensor);\n PRINT_SIZE(caffe2::CPUContext);\n PRINT_SIZE(caffe2::HIPContext);\n PRINT_SIZE(caffe2::OperatorBase);\n PRINT_SIZE(caffe2::OperatorDef);\n PRINT_SIZE(caffe2::Operator<caffe2::CPUContext>);\n PRINT_SIZE(caffe2::Operator<caffe2::HIPContext>);\n PRINT_SIZE(caffe2::TypeMeta);\n PRINT_SIZE(caffe2::Workspace);\n return 0;\n}\n###"261 },262 {263 "cuda": "\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n#include <c10/cuda/CUDADeviceAssertion.h>\n#include <c10/cuda/CUDAException.h>\n#include <c10/cuda/CUDAFunctions.h>\n#include <c10/cuda/CUDAStream.h>\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\nusing ::testing::HasSubstr;\nvoid did_not_fail_diagnostics() {\n std::cerr\n << \"c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = \"\n << c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime\n << std::endl;\n std::cerr\n << \"c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_compile_time = \"\n << c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_compile_time\n << std::endl;\n std::cerr\n << \"c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().do_all_devices_support_managed_memory = \"\n << c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref()\n .do_all_devices_support_managed_memory\n << std::endl;\n}\n\n__global__ void cuda_always_fail_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\nvoid cuda_device_assertions_1_var_test() {\n const auto stream = c10::cuda::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n cuda_always_fail_assertion_kernel, 1, 1, 0, stream, 1);\n try {\n c10::cuda::device_synchronize();\n did_not_fail_diagnostics();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\"CUDA device-side assertion failures were found on GPU #0!\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"Thread ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Block ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Name of kernel launched that led to failure = cuda_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\nTEST(CUDATest, cuda_device_assertions_1_var_test) {\n#ifdef TORCH_USE_CUDA_DSA\n c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n did_not_fail_diagnostics();\n cuda_device_assertions_1_var_test();\n#else\n GTEST_SKIP() << \"CUDA device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n\n###",264 "hip": " \n#include \"hip/hip_runtime.h\"\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n#include <c10/hip/HIPDeviceAssertion.h>\n#include <c10/hip/HIPException.h>\n#include <c10/hip/HIPFunctions.h>\n#include <c10/hip/HIPStream.h>\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\nusing ::testing::HasSubstr;\nvoid did_not_fail_diagnostics() {\n std::cerr\n << \"c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = \"\n << c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime\n << std::endl;\n std::cerr\n << \"c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_compile_time = \"\n << c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_compile_time\n << std::endl;\n std::cerr\n << \"c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().do_all_devices_support_managed_memory = \"\n << c10::hip::HIPKernelLaunchRegistry::get_singleton_ref()\n .do_all_devices_support_managed_memory\n << std::endl;\n}\n\n__global__ void hip_always_fail_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\nvoid hip_device_assertions_1_var_test() {\n const auto stream = c10::hip::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n hip_always_fail_assertion_kernel, 1, 1, 0, stream, 1);\n try {\n c10::hip::device_synchronize();\n did_not_fail_diagnostics();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\"HIP device-side assertion failures were found on GPU #0!\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"Thread ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Block ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Name of kernel launched that led to failure = hip_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\nTEST(HIPTest, hip_device_assertions_1_var_test) {\n#ifdef TORCH_USE_HIP_DSA\n c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n did_not_fail_diagnostics();\n hip_device_assertions_1_var_test();\n#else\n GTEST_SKIP() << \"HIP device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}###"265 },266 {267 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n\nnamespace at::native {\n\ntemplate<typename scalar_t>\nstruct AbsFunctor {\n __device__ __forceinline__ scalar_t operator() (const scalar_t a) const {\n return std::abs(a);\n }\n};\n\nCONSTEXPR_EXCEPT_WIN_CUDA char abs_name[] = \"abs_kernel\";\nvoid abs_kernel_cuda(TensorIteratorBase& iter) {\n auto dtype = iter.dtype();\n if (at::isComplexType(dtype)) {\n#if AT_USE_JITERATOR()\n static const auto abs_string = jiterator_stringify(\n template <typename T> T abs_kernel(T x) { return std::abs(x); });\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, dtype, \"abs_cuda\", [&]() {\n jitted_gpu_kernel<\n /*name=*/abs_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, abs_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, dtype, \"abs_cuda\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n gpu_kernel(iter, AbsFunctor<opmath_t>());\n });\n#endif\n } else {\n AT_DISPATCH_ALL_TYPES_AND3(\n ScalarType::Half,\n ScalarType::BFloat16,\n ScalarType::Bool,\n iter.dtype(),\n \"abs_cuda\",\n [&]() { gpu_kernel(iter, AbsFunctor<scalar_t>()); });\n }\n}\n\n REGISTER_DISPATCH(abs_stub, &abs_kernel_cuda);\n\n} // namespace at::native\n\n\n###",268 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n\nnamespace at::native {\n\ntemplate<typename scalar_t>\nstruct AbsFunctor {\n __device__ __forceinline__ scalar_t operator() (const scalar_t a) const {\n return std::abs(a);\n }\n};\n\nCONSTEXPR_EXCEPT_WIN_HIP char abs_name[] = \"abs_kernel\";\nvoid abs_kernel_hip(TensorIteratorBase& iter) {\n auto dtype = iter.dtype();\n if (at::isComplexType(dtype)) {\n#if AT_USE_JITERATOR()\n static const auto abs_string = jiterator_stringify(\n template <typename T> T abs_kernel(T x) { return std::abs(x); });\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, dtype, \"abs_hip\", [&]() {\n jitted_gpu_kernel<\n /*name=*/abs_name,\n /*return_dtype=*/scalar_t,\n /*common_dtype=*/scalar_t,\n /*arity=*/1>(iter, abs_string);\n });\n#else\n AT_DISPATCH_COMPLEX_TYPES_AND(kComplexHalf, dtype, \"abs_hip\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n gpu_kernel(iter, AbsFunctor<opmath_t>());\n });\n#endif\n } else {\n AT_DISPATCH_ALL_TYPES_AND3(\n ScalarType::Half,\n ScalarType::BFloat16,\n ScalarType::Bool,\n iter.dtype(),\n \"abs_hip\",\n [&]() { gpu_kernel(iter, AbsFunctor<scalar_t>()); });\n }\n}\n\n REGISTER_DISPATCH(abs_stub, &abs_kernel_hip);\n\n} // namespace at::native\n###"269 },270 {271 "cuda": "\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n#include <c10/cuda/CUDADeviceAssertion.h>\n#include <c10/cuda/CUDAException.h>\n#include <c10/cuda/CUDAFunctions.h>\n#include <c10/cuda/CUDAStream.h>\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\nusing ::testing::HasSubstr;\n\n__global__ void cuda_multiple_vars_always_fail_assertion_kernel(\n const int a, const int b, const int c, const int d, TORCH_DSA_KERNEL_ARGS) {\n int i = a + b + c + d;\n if (i != 0) {\n CUDA_KERNEL_ASSERT2(i == -i);\n } else {\n CUDA_KERNEL_ASSERT2(i == i + 1);\n }\n}\n\n__global__ void cuda_always_fail_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\nvoid cuda_device_assertions_catches_stream() {\n const auto stream = c10::cuda::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n cuda_multiple_vars_always_fail_assertion_kernel, 1, 1, 0, stream, 1, 2, 3, 4 \n );\n try {\n c10::cuda::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\"# of GPUs this process interacted with = 1\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"CUDA device-side assertion failures were found on GPU #0!\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"Thread ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Block ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Name of kernel launched that led to failure = cuda_multiple_vars_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\nTEST(CUDATest, cuda_device_assertions_catches_stream) {\n#ifdef TORCH_USE_CUDA_DSA\n c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n cuda_device_assertions_catches_stream();\n#else\n GTEST_SKIP() << \"CUDA device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n\n###",272 "hip": " \n#include \"hip/hip_runtime.h\"\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n#include <c10/hip/HIPDeviceAssertion.h>\n#include <c10/hip/HIPException.h>\n#include <c10/hip/HIPFunctions.h>\n#include <c10/hip/HIPStream.h>\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\nusing ::testing::HasSubstr;\n\n__global__ void hip_multiple_vars_always_fail_assertion_kernel(\n const int a, const int b, const int c, const int d, TORCH_DSA_KERNEL_ARGS) {\n int i = a + b + c + d;\n if (i != 0) {\n CUDA_KERNEL_ASSERT2(i == -i);\n } else {\n CUDA_KERNEL_ASSERT2(i == i + 1);\n }\n}\n\n__global__ void hip_always_fail_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\nvoid hip_device_assertions_catches_stream() {\n const auto stream = c10::hip::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n hip_multiple_vars_always_fail_assertion_kernel, 1, 1, 0, stream, 1, 2, 3, 4 \n );\n try {\n c10::hip::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\"# of GPUs this process interacted with = 1\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"HIP device-side assertion failures were found on GPU #0!\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"Thread ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Block ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Name of kernel launched that led to failure = hip_multiple_vars_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str, HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\nTEST(HIPTest, hip_device_assertions_catches_stream) {\n#ifdef TORCH_USE_HIP_DSA\n c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n hip_device_assertions_catches_stream();\n#else\n GTEST_SKIP() << \"HIP device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}###"273 },274 {275 "cuda": "\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n\n#include <c10/cuda/CUDADeviceAssertion.h>\n#include <c10/cuda/CUDAException.h>\n#include <c10/cuda/CUDAFunctions.h>\n#include <c10/cuda/CUDAStream.h>\n\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\n\nusing ::testing::HasSubstr;\n\n/**\n * Device kernel that takes 2 arguments\n * @param bad_thread represents the thread we want to trigger assertion on.\n * @param bad_block represents the block we want to trigger assertion on.\n * This kernel will only trigger a device side assertion for <<bad_block,\n * bad_thread>> pair. all the other blocks and threads pairs will basically be\n * no-op.\n */\n__global__ void cuda_device_assertions_fail_on_thread_block_kernel(\n const int bad_thread,\n const int bad_block,\n TORCH_DSA_KERNEL_ARGS) {\n if (threadIdx.x == bad_thread && blockIdx.x == bad_block) {\n CUDA_KERNEL_ASSERT2(false); // This comparison necessarily needs to fail\n }\n}\n\n/**\n * TEST: Triggering device side assertion on only 1 thread from <<<1024,128>>>\n * grid. kernel used is unique, it take 2 parameters to tell which particular\n * block and thread it should assert, all the other threads of the kernel will\n * be basically no-op.\n */\nvoid cuda_device_assertions_catches_thread_and_block_and_device() {\n const auto stream = c10::cuda::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n cuda_device_assertions_fail_on_thread_block_kernel,\n 1024, /* Blocks */\n 128, /* Threads */\n 0, /* Shared mem */\n stream, /* Stream */\n 29, /* bad thread */\n 937 /* bad block */\n );\n\n try {\n c10::cuda::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\"Thread ID that failed assertion = [29,0,0]\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"Block ID that failed assertion = [937,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Name of kernel launched that led to failure = cuda_device_assertions_fail_on_thread_block_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\n\nTEST(CUDATest, cuda_device_assertions_catches_thread_and_block_and_device) {\n#ifdef TORCH_USE_CUDA_DSA\n c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n cuda_device_assertions_catches_thread_and_block_and_device();\n#else\n GTEST_SKIP() << \"CUDA device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n\n\n###",276 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n\n#include <c10/hip/HIPDeviceAssertion.h>\n#include <c10/hip/HIPException.h>\n#include <c10/hip/HIPFunctions.h>\n#include <c10/hip/HIPStream.h>\n\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\n\nusing ::testing::HasSubstr;\n\n/**\n * Device kernel that takes 2 arguments\n * @param bad_thread represents the thread we want to trigger assertion on.\n * @param bad_block represents the block we want to trigger assertion on.\n * This kernel will only trigger a device side assertion for <<bad_block,\n * bad_thread>> pair. all the other blocks and threads pairs will basically be\n * no-op.\n */\n__global__ void hip_device_assertions_fail_on_thread_block_kernel(\n const int bad_thread,\n const int bad_block,\n TORCH_DSA_KERNEL_ARGS) {\n if (threadIdx.x == bad_thread && blockIdx.x == bad_block) {\n CUDA_KERNEL_ASSERT2(false); // This comparison necessarily needs to fail\n }\n}\n\n/**\n * TEST: Triggering device side assertion on only 1 thread from <<<1024,128>>>\n * grid. kernel used is unique, it take 2 parameters to tell which particular\n * block and thread it should assert, all the other threads of the kernel will\n * be basically no-op.\n */\nvoid hip_device_assertions_catches_thread_and_block_and_device() {\n const auto stream = c10::hip::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n hip_device_assertions_fail_on_thread_block_kernel,\n 1024, /* Blocks */\n 128, /* Threads */\n 0, /* Shared mem */\n stream, /* Stream */\n 29, /* bad thread */\n 937 /* bad block */\n );\n\n try {\n c10::hip::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\"Thread ID that failed assertion = [29,0,0]\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"Block ID that failed assertion = [937,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Name of kernel launched that led to failure = hip_device_assertions_fail_on_thread_block_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\n\nTEST(HIPTest, hip_device_assertions_catches_thread_and_block_and_device) {\n#ifdef TORCH_USE_HIP_DSA\n c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n hip_device_assertions_catches_thread_and_block_and_device();\n#else\n GTEST_SKIP() << \"HIP device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n###"277 },278 {279 "cuda": "\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n#include <c10/cuda/CUDADeviceAssertion.h>\n#include <c10/cuda/CUDAException.h>\n#include <c10/cuda/CUDAFunctions.h>\n#include <c10/cuda/CUDAStream.h>\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\nusing ::testing::HasSubstr;\nconst auto max_assertions_failure_str =\n \"Assertion failure \" + std::to_string(C10_CUDA_DSA_ASSERTION_COUNT - 1);\n\n__global__ void cuda_always_fail_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\n__global__ void cuda_always_succeed_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a == a);\n}\n\n#ifndef _MSC_VER\n\nvoid cuda_device_assertions_from_2_processes() {\n const auto n1 = fork();\n if (n1 == 0) {\n \n \n \n TORCH_DSA_KERNEL_LAUNCH(\n cuda_always_fail_assertion_kernel, 1, 1, 0, c10::cuda::getStreamFromPool(), 1);\n try {\n c10::cuda::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\n \"1 CUDA device-side assertion failures were found on GPU #0!\"));\n }\n \n std::this_thread::sleep_for(std::chrono::milliseconds(3000));\n } else {\n \n \n \n std::this_thread::sleep_for(std::chrono::milliseconds(2000));\n TORCH_DSA_KERNEL_LAUNCH(\n cuda_always_succeed_assertion_kernel, 1, 1, 0, c10::cuda::getStreamFromPool(), 1);\n try {\n c10::cuda::device_synchronize();\n } catch (const c10::Error& err) {\n ASSERT_TRUE(false); \n }\n \n exit(0);\n }\n}\nTEST(CUDATest, cuda_device_assertions_from_2_processes) {\n#ifdef TORCH_USE_CUDA_DSA\n c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n cuda_device_assertions_from_2_processes();\n#else\n GTEST_SKIP() << \"CUDA device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n#else\n#endif\n\n###",280 "hip": " \n#include \"hip/hip_runtime.h\"\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n#include <c10/hip/HIPDeviceAssertion.h>\n#include <c10/hip/HIPException.h>\n#include <c10/hip/HIPFunctions.h>\n#include <c10/hip/HIPStream.h>\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\nusing ::testing::HasSubstr;\nconst auto max_assertions_failure_str =\n \"Assertion failure \" + std::to_string(C10_HIP_DSA_ASSERTION_COUNT - 1);\n\n__global__ void hip_always_fail_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\n__global__ void hip_always_succeed_assertion_kernel(\n const int a, TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a == a);\n}\n\n#ifndef _MSC_VER\n\nvoid hip_device_assertions_from_2_processes() {\n const auto n1 = fork();\n if (n1 == 0) {\n \n \n \n TORCH_DSA_KERNEL_LAUNCH(\n hip_always_fail_assertion_kernel, 1, 1, 0, c10::hip::getStreamFromPool(), 1);\n try {\n c10::hip::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(\n err_str, HasSubstr(\n \"1 HIP device-side assertion failures were found on GPU #0!\"));\n }\n \n std::this_thread::sleep_for(std::chrono::milliseconds(3000));\n } else {\n \n \n \n std::this_thread::sleep_for(std::chrono::milliseconds(2000));\n TORCH_DSA_KERNEL_LAUNCH(\n hip_always_succeed_assertion_kernel, 1, 1, 0, c10::hip::getStreamFromPool(), 1);\n try {\n c10::hip::device_synchronize();\n } catch (const c10::Error& err) {\n ASSERT_TRUE(false); \n }\n \n exit(0);\n }\n}\nTEST(HIPTest, hip_device_assertions_from_2_processes) {\n#ifdef TORCH_USE_HIP_DSA\n c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n hip_device_assertions_from_2_processes();\n#else\n GTEST_SKIP() << \"HIP device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n#else\n#endif###"281 },282 {283 "cuda": "\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n\n#include <c10/cuda/CUDADeviceAssertion.h>\n#include <c10/cuda/CUDAException.h>\n#include <c10/cuda/CUDAFunctions.h>\n#include <c10/cuda/CUDAStream.h>\n\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\n\nusing ::testing::HasSubstr;\n\nconst auto max_assertions_failure_str =\n \"Assertion failure \" + std::to_string(C10_CUDA_DSA_ASSERTION_COUNT - 1);\n\n/**\n * Device kernel that takes a single integer parameter as argument and\n * will always trigger a device side assertion.\n */\n__global__ void cuda_always_fail_assertion_kernel(\n const int a,\n TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\n/**\n * TEST: Triggering device side assertion from multiple block but single thread\n * <<<10,128>>>. Here we are triggering assertion on 10 blocks, each with only\n * 128 thread.\n */\nvoid cuda_device_assertions_multiple_writes_from_blocks_and_threads() {\n bool run_threads = false;\n\n // Create a function to launch kernel that waits for a signal, to try to\n // ensure everything is happening simultaneously\n const auto launch_the_kernel = [&]() {\n // Busy loop waiting for the signal to go\n while (!run_threads) {\n }\n\n TORCH_DSA_KERNEL_LAUNCH(\n cuda_always_fail_assertion_kernel,\n 10, /* Blocks */\n 128, /* Threads */\n 0, /* Shared mem */\n c10::cuda::getCurrentCUDAStream(), /* Stream */\n 1);\n };\n\n // Spin up a bunch of busy-looping threads\n std::vector<std::thread> threads;\n for (int i = 0; i < 10; i++) {\n threads.emplace_back(launch_the_kernel);\n }\n\n // Paranoid - wait for all the threads to get setup\n std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n // Mash\n run_threads = true;\n\n // Clean-up\n for (auto& x : threads) {\n x.join();\n }\n\n try {\n c10::cuda::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(err_str, HasSubstr(max_assertions_failure_str));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Name of kernel launched that led to failure = cuda_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n }\n}\n\nTEST(CUDATest, cuda_device_assertions_multiple_writes_from_blocks_and_threads) {\n#ifdef TORCH_USE_CUDA_DSA\n c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n cuda_device_assertions_multiple_writes_from_blocks_and_threads();\n#else\n GTEST_SKIP() << \"CUDA device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n\n\n###",284 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n\n#include <c10/hip/HIPDeviceAssertion.h>\n#include <c10/hip/HIPException.h>\n#include <c10/hip/HIPFunctions.h>\n#include <c10/hip/HIPStream.h>\n\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\n\nusing ::testing::HasSubstr;\n\nconst auto max_assertions_failure_str =\n \"Assertion failure \" + std::to_string(C10_HIP_DSA_ASSERTION_COUNT - 1);\n\n/**\n * Device kernel that takes a single integer parameter as argument and\n * will always trigger a device side assertion.\n */\n__global__ void hip_always_fail_assertion_kernel(\n const int a,\n TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\n/**\n * TEST: Triggering device side assertion from multiple block but single thread\n * <<<10,128>>>. Here we are triggering assertion on 10 blocks, each with only\n * 128 thread.\n */\nvoid hip_device_assertions_multiple_writes_from_blocks_and_threads() {\n bool run_threads = false;\n\n // Create a function to launch kernel that waits for a signal, to try to\n // ensure everything is happening simultaneously\n const auto launch_the_kernel = [&]() {\n // Busy loop waiting for the signal to go\n while (!run_threads) {\n }\n\n TORCH_DSA_KERNEL_LAUNCH(\n hip_always_fail_assertion_kernel,\n 10, /* Blocks */\n 128, /* Threads */\n 0, /* Shared mem */\n c10::hip::getCurrentHIPStream(), /* Stream */\n 1);\n };\n\n // Spin up a bunch of busy-looping threads\n std::vector<std::thread> threads;\n for (int i = 0; i < 10; i++) {\n threads.emplace_back(launch_the_kernel);\n }\n\n // Paranoid - wait for all the threads to get setup\n std::this_thread::sleep_for(std::chrono::milliseconds(100));\n\n // Mash\n run_threads = true;\n\n // Clean-up\n for (auto& x : threads) {\n x.join();\n }\n\n try {\n c10::hip::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(err_str, HasSubstr(max_assertions_failure_str));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Name of kernel launched that led to failure = hip_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n }\n}\n\nTEST(HIPTest, hip_device_assertions_multiple_writes_from_blocks_and_threads) {\n#ifdef TORCH_USE_HIP_DSA\n c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n hip_device_assertions_multiple_writes_from_blocks_and_threads();\n#else\n GTEST_SKIP() << \"HIP device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n###"285 },286 {287 "cuda": "\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n\n#include <c10/cuda/CUDADeviceAssertion.h>\n#include <c10/cuda/CUDAException.h>\n#include <c10/cuda/CUDAFunctions.h>\n#include <c10/cuda/CUDAStream.h>\n\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\n\nusing ::testing::HasSubstr;\n\nconst auto max_assertions_failure_str =\n \"Assertion failure \" + std::to_string(C10_CUDA_DSA_ASSERTION_COUNT - 1);\n\n/**\n * Device kernel that takes a single integer parameter as argument and\n * will always trigger a device side assertion.\n */\n__global__ void cuda_always_fail_assertion_kernel(\n const int a,\n TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\n/**\n * TEST: Triggering device side assertion from single block and multiple threads\n * <<<1,128>>>. Once the very first thread asserts all the other threads will\n * basically be in bad state and the block id with failed assertion would be\n * [0,0,0].\n */\nvoid cuda_device_assertions_multiple_writes_from_same_block() {\n const auto stream = c10::cuda::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n cuda_always_fail_assertion_kernel,\n 1, /* Blocks */\n 128, /* Threads */\n 0, /* Shared mem */\n stream, /* Stream */\n 1);\n\n try {\n c10::cuda::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(err_str, HasSubstr(max_assertions_failure_str));\n ASSERT_THAT(err_str, HasSubstr(\"Block ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Name of kernel launched that led to failure = cuda_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\n\nTEST(CUDATest, cuda_device_assertions_multiple_writes_from_same_block) {\n#ifdef TORCH_USE_CUDA_DSA\n c10::cuda::CUDAKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n cuda_device_assertions_multiple_writes_from_same_block();\n#else\n GTEST_SKIP() << \"CUDA device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n\n\n###",288 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <gmock/gmock.h>\n#include <gtest/gtest.h>\n\n#include <c10/hip/HIPDeviceAssertion.h>\n#include <c10/hip/HIPException.h>\n#include <c10/hip/HIPFunctions.h>\n#include <c10/hip/HIPStream.h>\n\n#include <chrono>\n#include <iostream>\n#include <string>\n#include <thread>\n\nusing ::testing::HasSubstr;\n\nconst auto max_assertions_failure_str =\n \"Assertion failure \" + std::to_string(C10_HIP_DSA_ASSERTION_COUNT - 1);\n\n/**\n * Device kernel that takes a single integer parameter as argument and\n * will always trigger a device side assertion.\n */\n__global__ void hip_always_fail_assertion_kernel(\n const int a,\n TORCH_DSA_KERNEL_ARGS) {\n CUDA_KERNEL_ASSERT2(a != a);\n}\n\n/**\n * TEST: Triggering device side assertion from single block and multiple threads\n * <<<1,128>>>. Once the very first thread asserts all the other threads will\n * basically be in bad state and the block id with failed assertion would be\n * [0,0,0].\n */\nvoid hip_device_assertions_multiple_writes_from_same_block() {\n const auto stream = c10::hip::getStreamFromPool();\n TORCH_DSA_KERNEL_LAUNCH(\n hip_always_fail_assertion_kernel,\n 1, /* Blocks */\n 128, /* Threads */\n 0, /* Shared mem */\n stream, /* Stream */\n 1);\n\n try {\n c10::hip::device_synchronize();\n throw std::runtime_error(\"Test didn't fail, but should have.\");\n } catch (const c10::Error& err) {\n const auto err_str = std::string(err.what());\n ASSERT_THAT(err_str, HasSubstr(max_assertions_failure_str));\n ASSERT_THAT(err_str, HasSubstr(\"Block ID that failed assertion = [0,0,0]\"));\n ASSERT_THAT(err_str, HasSubstr(\"Device that launched kernel = 0\"));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Name of kernel launched that led to failure = hip_always_fail_assertion_kernel\"));\n ASSERT_THAT(\n err_str, HasSubstr(\"File containing kernel launch = \" __FILE__));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Function containing kernel launch = \" +\n std::string(__FUNCTION__)));\n ASSERT_THAT(\n err_str,\n HasSubstr(\n \"Stream kernel was launched on = \" + std::to_string(stream.id())));\n }\n}\n\nTEST(HIPTest, hip_device_assertions_multiple_writes_from_same_block) {\n#ifdef TORCH_USE_HIP_DSA\n c10::hip::HIPKernelLaunchRegistry::get_singleton_ref().enabled_at_runtime = true;\n hip_device_assertions_multiple_writes_from_same_block();\n#else\n GTEST_SKIP() << \"HIP device-side assertions (DSA) was not enabled at compile time.\";\n#endif\n}\n###"289 },290 {291 "cuda": "\n#include \"caffe2/contrib/aten/aten_op.h\"\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(ATen, ATenOp<CUDAContext>);\ntemplate<>\nat::Backend ATenOp<CUDAContext>::backend() const {\n return at::Backend::CUDA;\n}\n\n}\n\n\n###",292 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/contrib/aten/aten_op.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(ATen, ATenOp<HIPContext>);\ntemplate<>\nat::Backend ATenOp<HIPContext>::backend() const {\n return at::Backend::HIP;\n}\n\n}\n###"293 },294 {295 "cuda": "\n#include \"caffe2/contrib/gloo/broadcast_ops.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\n#include <gloo/cuda_broadcast_one_to_all.h>\n\nnamespace caffe2 {\nnamespace gloo {\n\ntemplate <class Context>\nvoid BroadcastOp<Context>::initializeAlgorithm() {\n if (init_.template IsType<float>()) {\n algorithm_.reset(new ::gloo::CudaBroadcastOneToAll<float>(\n init_.context, init_.template getOutputs<float>(), init_.size, root_));\n } else if (init_.template IsType<long>()) {\n algorithm_.reset(new ::gloo::CudaBroadcastOneToAll<long>(\n init_.context, init_.template getOutputs<long>(), init_.size, root_));\n } else if (init_.template IsType<int>()) {\n algorithm_.reset(new ::gloo::CudaBroadcastOneToAll<int>(\n init_.context, init_.template getOutputs<int>(), init_.size, root_));\n } else if (init_.template IsType<at::Half>()) {\n algorithm_.reset(new ::gloo::CudaBroadcastOneToAll<::gloo::float16>(\n init_.context,\n init_.template getOutputs<::gloo::float16>(),\n init_.size,\n root_));\n } else {\n CAFFE_ENFORCE(false, \"Unhandled type: \", init_.meta.name());\n }\n}\n\nnamespace {\n\nREGISTER_CUDA_OPERATOR_WITH_ENGINE(Broadcast, GLOO, BroadcastOp<CUDAContext>);\n\n} // namespace\n} // namespace gloo\n} // namespace caffe2\n\n\n###",296 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/contrib/gloo/broadcast_ops.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\n#include <gloo/hip_broadcast_one_to_all.h>\n\nnamespace caffe2 {\nnamespace gloo {\n\ntemplate <class Context>\nvoid BroadcastOp<Context>::initializeAlgorithm() {\n if (init_.template IsType<float>()) {\n algorithm_.reset(new ::gloo::HipBroadcastOneToAll<float>(\n init_.context, init_.template getOutputs<float>(), init_.size, root_));\n } else if (init_.template IsType<long>()) {\n algorithm_.reset(new ::gloo::HipBroadcastOneToAll<long>(\n init_.context, init_.template getOutputs<long>(), init_.size, root_));\n } else if (init_.template IsType<int>()) {\n algorithm_.reset(new ::gloo::HipBroadcastOneToAll<int>(\n init_.context, init_.template getOutputs<int>(), init_.size, root_));\n } else if (init_.template IsType<at::Half>()) {\n algorithm_.reset(new ::gloo::HipBroadcastOneToAll<::gloo::float16>(\n init_.context,\n init_.template getOutputs<::gloo::float16>(),\n init_.size,\n root_));\n } else {\n CAFFE_ENFORCE(false, \"Unhandled type: \", init_.meta.name());\n }\n}\n\nnamespace {\n\nREGISTER_HIP_OPERATOR_WITH_ENGINE(Broadcast, GLOO, BroadcastOp<HIPContext>);\n\n} // namespace\n} // namespace gloo\n} // namespace caffe2\n###"297 },298 {299 "cuda": "\n#include \"caffe2/contrib/gloo/common_world_ops.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\n#include <gloo/cuda.h>\n#include <gloo/transport/tcp/device.h>\n\nnamespace caffe2 {\nnamespace gloo {\n\ntemplate <>\nvoid CreateCommonWorld<CUDAContext>::initializeForContext() {\n static std::once_flag once;\n std::call_once(once, [&]() {\n // This is the first time we call Gloo code for a CUDAContext.\n // Share Caffe2 CUDA mutex with Gloo.\n ::gloo::CudaShared::setMutex(&CUDAContext::mutex());\n });\n}\n\nnamespace {\n\nREGISTER_CUDA_OPERATOR_WITH_ENGINE(\n CreateCommonWorld,\n GLOO,\n CreateCommonWorld<CUDAContext>);\n\nREGISTER_CUDA_OPERATOR_WITH_ENGINE(\n CloneCommonWorld,\n GLOO,\n CloneCommonWorld<CUDAContext>);\n\n} // namespace\n} // namespace gloo\n} // namespace caffe2\n\n\n###",300 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/contrib/gloo/common_world_ops.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\n#include <gloo/hip.h>\n#include <gloo/transport/tcp/device.h>\n\nnamespace caffe2 {\nnamespace gloo {\n\ntemplate <>\nvoid CreateCommonWorld<HIPContext>::initializeForContext() {\n static std::once_flag once;\n std::call_once(once, [&]() {\n // This is the first time we call Gloo code for a HIPContext.\n // Share Caffe2 HIP mutex with Gloo.\n ::gloo::HipShared::setMutex(&HIPContext::mutex());\n });\n}\n\nnamespace {\n\nREGISTER_HIP_OPERATOR_WITH_ENGINE(\n CreateCommonWorld,\n GLOO,\n CreateCommonWorld<HIPContext>);\n\nREGISTER_HIP_OPERATOR_WITH_ENGINE(\n CloneCommonWorld,\n GLOO,\n CloneCommonWorld<HIPContext>);\n\n} // namespace\n} // namespace gloo\n} // namespace caffe2\n###"301 },302 {303 "cuda": "\n#pragma once\n\n#include <cstddef>\n\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/logging.h\"\n\n#include <nccl.h>\n#include <unordered_map>\n\n#define NCCL_VERSION_MIN(major, minor, patch) \\\n ((NCCL_MAJOR > major) || \\\n ((NCCL_MAJOR == major) && \\\n ((NCCL_MINOR > minor) || \\\n ((NCCL_MINOR == minor) && (NCCL_PATCH >= patch)))))\n\nnamespace caffe2 {\nnamespace nccl {\n\n#define CAFFE_NCCL_CHECK(condition) \\\n do { \\\n ncclResult_t status = (condition); \\\n CAFFE_ENFORCE_EQ( \\\n status, \\\n ncclSuccess, \\\n \" \", \\\n \"Error at: \", \\\n __FILE__, \\\n __LINE__, \\\n \": \", \\\n ncclGetErrorString(status)); \\\n } while (0)\n\nstruct NCCLElement {\n const TensorCUDA* src{nullptr};\n TensorCUDA* dst{nullptr};\n int device{0};\n};\n\nstruct NCCLExecution {\n int stream_gpu_id{0};\n cudaStream_t stream{nullptr};\n std::vector<NCCLElement> elements;\n size_t root{0};\n};\n\n// Called when the last NCCL op is destructed and all lazily created\n// NCCLContext instances can safely be destroyed.\nvoid destroyContexts();\n\ntemplate <typename T>\nclass NCCL {\n public:\n static void AllReduce(const NCCLExecution& ex);\n static void Broadcast(const NCCLExecution& ex);\n static void Reduce(const NCCLExecution& ex);\n static void AllGather(const NCCLExecution& ex);\n static void ReduceScatter(const NCCLExecution& ex);\n};\n\n} // namespace nccl\n} // namespace caffe2\n\n\n###",304 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <cstddef>\n\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/logging.h\"\n\n#include <rccl.h>\n#include <unordered_map>\n\n#define NCCL_VERSION_MIN(major, minor, patch) \\\n ((NCCL_MAJOR > major) || \\\n ((NCCL_MAJOR == major) && \\\n ((NCCL_MINOR > minor) || \\\n ((NCCL_MINOR == minor) && (NCCL_PATCH >= patch)))))\n\nnamespace caffe2 {\nnamespace nccl {\n\n#define CAFFE_NCCL_CHECK(condition) \\\n do { \\\n ncclResult_t status = (condition); \\\n CAFFE_ENFORCE_EQ( \\\n status, \\\n ncclSuccess, \\\n \" \", \\\n \"Error at: \", \\\n __FILE__, \\\n __LINE__, \\\n \": \", \\\n ncclGetErrorString(status)); \\\n } while (0)\n\nstruct NCCLElement {\n const TensorHIP* src{nullptr};\n TensorHIP* dst{nullptr};\n int device{0};\n};\n\nstruct NCCLExecution {\n int stream_gpu_id{0};\n hipStream_t stream{nullptr};\n std::vector<NCCLElement> elements;\n size_t root{0};\n};\n\n// Called when the last NCCL op is destructed and all lazily created\n// NCCLContext instances can safely be destroyed.\nvoid destroyContexts();\n\ntemplate <typename T>\nclass NCCL {\n public:\n static void AllReduce(const NCCLExecution& ex);\n static void Broadcast(const NCCLExecution& ex);\n static void Reduce(const NCCLExecution& ex);\n static void AllGather(const NCCLExecution& ex);\n static void ReduceScatter(const NCCLExecution& ex);\n};\n\n} // namespace nccl\n} // namespace caffe2\n###"305 },306 {307 "cuda": "\n#include \"caffe2/core/blob.h\"\n#include \"caffe2/core/blob_serialization.h\"\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\nREGISTER_BLOB_DESERIALIZER(TensorCUDA, TensorDeserializer);\n}\n} // namespace caffe2\n\n\n###",308 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/blob.h\"\n#include \"caffe2/core/blob_serialization.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\nREGISTER_BLOB_DESERIALIZER(TensorHIP, TensorDeserializer);\n}\n} // namespace caffe2\n###"309 },310 {311 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid elu_kernel(\n TensorIteratorBase& iter,\n const Scalar& alpha,\n const Scalar& scale,\n const Scalar& input_scale) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"elu_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negcoef = alpha.to<opmath_t>() * scale.to<opmath_t>();\n auto poscoef = scale.to<opmath_t>();\n auto negiptcoef = input_scale.to<opmath_t>();\n gpu_kernel(\n iter,\n [negcoef, poscoef, negiptcoef] GPU_LAMBDA(scalar_t a) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n return aop > 0 ? aop * poscoef\n : std::expm1(aop * negiptcoef) * negcoef;\n });\n });\n}\n\nvoid elu_backward_kernel(\n TensorIteratorBase& iter,\n const Scalar& alpha,\n const Scalar& scale,\n const Scalar& input_scale,\n bool is_result) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"elu_backward_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negcoef = alpha.to<opmath_t>() * scale.to<opmath_t>();\n auto poscoef = scale.to<opmath_t>();\n auto negiptcoef = input_scale.to<opmath_t>();\n gpu_kernel(\n iter,\n [negcoef, poscoef, negiptcoef, is_result] GPU_LAMBDA(\n scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n\n if (is_result) {\n return bop <= 0 ? aop * negiptcoef * (bop + negcoef)\n : aop * poscoef;\n } else {\n return bop <= 0\n ? aop * negiptcoef * negcoef * std::exp(bop * negiptcoef)\n : aop * poscoef;\n }\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(elu_stub, &elu_kernel);\nREGISTER_DISPATCH(elu_backward_stub, &elu_backward_kernel);\n\n} // namespace at::native\n\n\n###",312 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid elu_kernel(\n TensorIteratorBase& iter,\n const Scalar& alpha,\n const Scalar& scale,\n const Scalar& input_scale) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"elu_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negcoef = alpha.to<opmath_t>() * scale.to<opmath_t>();\n auto poscoef = scale.to<opmath_t>();\n auto negiptcoef = input_scale.to<opmath_t>();\n gpu_kernel(\n iter,\n [negcoef, poscoef, negiptcoef] GPU_LAMBDA(scalar_t a) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n return aop > 0 ? aop * poscoef\n : std::expm1(aop * negiptcoef) * negcoef;\n });\n });\n}\n\nvoid elu_backward_kernel(\n TensorIteratorBase& iter,\n const Scalar& alpha,\n const Scalar& scale,\n const Scalar& input_scale,\n bool is_result) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"elu_backward_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negcoef = alpha.to<opmath_t>() * scale.to<opmath_t>();\n auto poscoef = scale.to<opmath_t>();\n auto negiptcoef = input_scale.to<opmath_t>();\n gpu_kernel(\n iter,\n [negcoef, poscoef, negiptcoef, is_result] GPU_LAMBDA(\n scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n\n if (is_result) {\n return bop <= 0 ? aop * negiptcoef * (bop + negcoef)\n : aop * poscoef;\n } else {\n return bop <= 0\n ? aop * negiptcoef * negcoef * ::exp(bop * negiptcoef)\n : aop * poscoef;\n }\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(elu_stub, &elu_kernel);\nREGISTER_DISPATCH(elu_backward_stub, &elu_backward_kernel);\n\n} // namespace at::native\n###"313 },314 {315 "cuda": "\n#include <gtest/gtest.h>\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/event.h\"\n\nnamespace caffe2 {\n\nTEST(EventCUDATest, EventBasics) {\n if (!HasCudaGPU())\n return;\n DeviceOption device_cpu;\n device_cpu.set_device_type(PROTO_CPU);\n DeviceOption device_cuda;\n device_cuda.set_device_type(PROTO_CUDA);\n\n CPUContext context_cpu(device_cpu);\n CUDAContext context_cuda(device_cuda);\n\n Event event_cpu(device_cpu);\n Event event_cuda(device_cuda);\n\n // CPU context and event interactions\n context_cpu.Record(&event_cpu);\n event_cpu.SetFinished();\n event_cpu.Finish();\n context_cpu.WaitEvent(event_cpu);\n\n event_cpu.Reset();\n event_cpu.Record(CPU, &context_cpu);\n event_cpu.SetFinished();\n event_cpu.Wait(CPU, &context_cpu);\n\n // CUDA context and event interactions\n context_cuda.SwitchToDevice();\n context_cuda.Record(&event_cuda);\n context_cuda.WaitEvent(event_cuda);\n event_cuda.Finish();\n\n event_cuda.Reset();\n event_cuda.Record(CUDA, &context_cuda);\n event_cuda.Wait(CUDA, &context_cuda);\n\n // CPU context waiting for CUDA event\n context_cpu.WaitEvent(event_cuda);\n\n // CUDA context waiting for CPU event\n context_cuda.WaitEvent(event_cpu);\n}\n\n} // namespace caffe2\n\n\n###",316 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <gtest/gtest.h>\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/event.h\"\n\nnamespace caffe2 {\n\nTEST(EventHIPTest, EventBasics) {\n if (!HasHipGPU())\n return;\n DeviceOption device_cpu;\n device_cpu.set_device_type(PROTO_CPU);\n DeviceOption device_hip;\n device_hip.set_device_type(PROTO_HIP);\n\n CPUContext context_cpu(device_cpu);\n HIPContext context_hip(device_hip);\n\n Event event_cpu(device_cpu);\n Event event_hip(device_hip);\n\n // CPU context and event interactions\n context_cpu.Record(&event_cpu);\n event_cpu.SetFinished();\n event_cpu.Finish();\n context_cpu.WaitEvent(event_cpu);\n\n event_cpu.Reset();\n event_cpu.Record(CPU, &context_cpu);\n event_cpu.SetFinished();\n event_cpu.Wait(CPU, &context_cpu);\n\n // HIP context and event interactions\n context_hip.SwitchToDevice();\n context_hip.Record(&event_hip);\n context_hip.WaitEvent(event_hip);\n event_hip.Finish();\n\n event_hip.Reset();\n event_hip.Record(HIP, &context_hip);\n event_hip.Wait(HIP, &context_hip);\n\n // CPU context waiting for HIP event\n context_cpu.WaitEvent(event_hip);\n\n // HIP context waiting for CPU event\n context_hip.WaitEvent(event_cpu);\n}\n\n} // namespace caffe2\n###"317 },318 {319 "cuda": "\n#include <string>\n\n#include <gtest/gtest.h>\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/operator.h\"\n\nnamespace caffe2 {\n\nclass JustTest : public OperatorBase {\n public:\n using OperatorBase::OperatorBase;\n bool Run(int /* unused */ /*stream_id*/) override {\n return true;\n }\n virtual std::string type() {\n return \"BASE\";\n }\n};\n\nclass JustTestCUDA : public JustTest {\n public:\n using JustTest::JustTest;\n bool Run(int /* unused */ /*stream_id*/) override {\n return true;\n }\n std::string type() override {\n return \"CUDA\";\n }\n};\n\nclass JustTestCUDNN : public JustTest {\n public:\n using JustTest::JustTest;\n bool Run(int /* unused */ /*stream_id*/) override {\n return true;\n }\n std::string type() override {\n return \"CUDNN\";\n }\n};\n\nOPERATOR_SCHEMA(JustTest).NumInputs(0, 1).NumOutputs(0, 1);\nREGISTER_CUDA_OPERATOR(JustTest, JustTestCUDA);\nREGISTER_CUDNN_OPERATOR(JustTest, JustTestCUDNN);\n\nTEST(EnginePrefTest, GPUDeviceDefaultPreferredEngines) {\n if (!HasCudaGPU())\n return;\n OperatorDef op_def;\n Workspace ws;\n op_def.mutable_device_option()->set_device_type(PROTO_CUDA);\n op_def.set_type(\"JustTest\");\n\n {\n const auto op = CreateOperator(op_def, &ws);\n EXPECT_NE(nullptr, op.get());\n // CUDNN should be taken as it's in the default global preferred engines\n // list\n EXPECT_EQ(static_cast<JustTest*>(op.get())->type(), \"CUDNN\");\n }\n}\n\n} // namespace caffe2\n\n\n###",320 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <string>\n\n#include <gtest/gtest.h>\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/operator.h\"\n\nnamespace caffe2 {\n\nclass JustTest : public OperatorBase {\n public:\n using OperatorBase::OperatorBase;\n bool Run(int /* unused */ /*stream_id*/) override {\n return true;\n }\n virtual std::string type() {\n return \"BASE\";\n }\n};\n\nclass JustTestHIP : public JustTest {\n public:\n using JustTest::JustTest;\n bool Run(int /* unused */ /*stream_id*/) override {\n return true;\n }\n std::string type() override {\n return \"HIP\";\n }\n};\n\nclass JustTestMIOPEN : public JustTest {\n public:\n using JustTest::JustTest;\n bool Run(int /* unused */ /*stream_id*/) override {\n return true;\n }\n std::string type() override {\n return \"MIOPEN\";\n }\n};\n\nOPERATOR_SCHEMA(JustTest).NumInputs(0, 1).NumOutputs(0, 1);\nREGISTER_HIP_OPERATOR(JustTest, JustTestHIP);\nREGISTER_MIOPEN_OPERATOR(JustTest, JustTestMIOPEN);\n\nTEST(EnginePrefTest, GPUDeviceDefaultPreferredEngines) {\n if (!HasHipGPU())\n return;\n OperatorDef op_def;\n Workspace ws;\n op_def.mutable_device_option()->set_device_type(PROTO_HIP);\n op_def.set_type(\"JustTest\");\n\n {\n const auto op = CreateOperator(op_def, &ws);\n EXPECT_NE(nullptr, op.get());\n // MIOPEN should be taken as it's in the default global preferred engines\n // list\n EXPECT_EQ(static_cast<JustTest*>(op.get())->type(), \"MIOPEN\");\n }\n}\n\n} // namespace caffe2\n###"321 },322 {323 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/db/create_db_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(CreateDB, CreateDBOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",324 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/db/create_db_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(CreateDB, CreateDBOp<HIPContext>);\n} // namespace caffe2\n###"325 },326 {327 "cuda": "\n#include \"caffe2/distributed/file_store_handler_op.h\"\n\n#if !defined(USE_ROCM)\n#include <caffe2/core/context_gpu.h>\n#else\n#include <caffe2/core/hip/context_gpu.h>\n#endif\n\nnamespace caffe2 {\n\n#if !defined(USE_ROCM)\nREGISTER_CUDA_OPERATOR(\n FileStoreHandlerCreate,\n FileStoreHandlerCreateOp<CUDAContext>);\n#else\nREGISTER_HIP_OPERATOR(\n FileStoreHandlerCreate,\n FileStoreHandlerCreateOp<HIPContext>);\n#endif\n\n} // namespace caffe2\n\n\n###",328 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/distributed/file_store_handler_op.h\"\n\n#if !defined(USE_ROCM)\n#include <caffe2/core/hip/context_gpu.h>\n#else\n#include <caffe2/core/hip/context_gpu.h>\n#endif\n\nnamespace caffe2 {\n\n#if !defined(USE_ROCM)\nREGISTER_HIP_OPERATOR(\n FileStoreHandlerCreate,\n FileStoreHandlerCreateOp<HIPContext>);\n#else\nREGISTER_HIP_OPERATOR(\n FileStoreHandlerCreate,\n FileStoreHandlerCreateOp<HIPContext>);\n#endif\n\n} // namespace caffe2\n###"329 },330 {331 "cuda": "\n#include \"caffe2/distributed/redis_store_handler_op.h\"\n\n#if !defined(USE_ROCM)\n#include <caffe2/core/context_gpu.h>\n#else\n#include <caffe2/core/hip/context_gpu.h>\n#endif\n\nnamespace caffe2 {\n\n#if !defined(USE_ROCM)\nREGISTER_CUDA_OPERATOR(\n RedisStoreHandlerCreate,\n RedisStoreHandlerCreateOp<CUDAContext>);\n#else\nREGISTER_HIP_OPERATOR(\n RedisStoreHandlerCreate,\n RedisStoreHandlerCreateOp<HIPContext>);\n#endif\n\n} // namespace caffe2\n\n\n###",332 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/distributed/redis_store_handler_op.h\"\n\n#if !defined(USE_ROCM)\n#include <caffe2/core/hip/context_gpu.h>\n#else\n#include <caffe2/core/hip/context_gpu.h>\n#endif\n\nnamespace caffe2 {\n\n#if !defined(USE_ROCM)\nREGISTER_HIP_OPERATOR(\n RedisStoreHandlerCreate,\n RedisStoreHandlerCreateOp<HIPContext>);\n#else\nREGISTER_HIP_OPERATOR(\n RedisStoreHandlerCreate,\n RedisStoreHandlerCreateOp<HIPContext>);\n#endif\n\n} // namespace caffe2\n###"333 },334 {335 "cuda": "\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/image/image_input_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool ImageInputOp<CUDAContext>::ApplyTransformOnGPU(\n const std::vector<std::int64_t>& dims,\n const c10::Device& type) {\n // GPU transform kernel allows explicitly setting output type\n if (output_type_ == TensorProto_DataType_FLOAT) {\n auto* image_output =\n OperatorBase::OutputTensor(0, dims, at::dtype<float>().device(type));\n TransformOnGPU<uint8_t, float, CUDAContext>(\n prefetched_image_on_device_,\n image_output,\n mean_gpu_,\n std_gpu_,\n &context_);\n } else if (output_type_ == TensorProto_DataType_FLOAT16) {\n auto* image_output =\n OperatorBase::OutputTensor(0, dims, at::dtype<at::Half>().device(type));\n TransformOnGPU<uint8_t, at::Half, CUDAContext>(\n prefetched_image_on_device_,\n image_output,\n mean_gpu_,\n std_gpu_,\n &context_);\n } else {\n return false;\n }\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(ImageInput, ImageInputOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",336 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/image/image_input_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool ImageInputOp<HIPContext>::ApplyTransformOnGPU(\n const std::vector<std::int64_t>& dims,\n const c10::Device& type) {\n // GPU transform kernel allows explicitly setting output type\n if (output_type_ == TensorProto_DataType_FLOAT) {\n auto* image_output =\n OperatorBase::OutputTensor(0, dims, at::dtype<float>().device(type));\n TransformOnGPU<uint8_t, float, HIPContext>(\n prefetched_image_on_device_,\n image_output,\n mean_gpu_,\n std_gpu_,\n &context_);\n } else if (output_type_ == TensorProto_DataType_FLOAT16) {\n auto* image_output =\n OperatorBase::OutputTensor(0, dims, at::dtype<at::Half>().device(type));\n TransformOnGPU<uint8_t, at::Half, HIPContext>(\n prefetched_image_on_device_,\n image_output,\n mean_gpu_,\n std_gpu_,\n &context_);\n } else {\n return false;\n }\n return true;\n}\n\nREGISTER_HIP_OPERATOR(ImageInput, ImageInputOp<HIPContext>);\n\n} // namespace caffe2\n###"337 },338 {339 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/image/transform_gpu.h\"\n#include \"caffe2/utils/conversions.h\"\n\n/**\n *\n * Copyright (c) 2016, NVIDIA CORPORATION, All rights reserved\n * Distributed under 2-clause BSD license; see accompanying LICENSE file\n *\n **/\n\nnamespace caffe2 {\n\nnamespace {\n\n// input in (int8, NHWC), output in (fp32, NCHW)\ntemplate <typename In, typename Out>\n__global__ void transform_kernel(\n const int C,\n const int H,\n const int W,\n const float* mean,\n const float* std,\n const In* in,\n Out* out) {\n const auto n = blockIdx.x;\n\n const auto nStride = C*H*W;\n\n // pointers to data for this image\n const In *const input_ptr = &in[n*nStride];\n Out *const output_ptr = &out[n*nStride];\n\n // either read or write uncoalesced - try reading\n for (int c=0; c < C; ++c) {\n for (int h=threadIdx.y; h < H; h += blockDim.y) {\n for (int w=threadIdx.x; w < W; w += blockDim.x) {\n const int in_idx = c + C*w + C*W*h; // HWC\n const int out_idx = c*H*W + h*W + w; // CHW\n\n output_ptr[out_idx] = convert::To<float,Out>(\n (convert::To<In,float>(input_ptr[in_idx])-mean[c]) * std[c]);\n }\n }\n }\n}\n\n}\n\ntemplate <typename T_IN, typename T_OUT, class Context>\n\nbool TransformOnGPU(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n Context* context) {\n const int N = X.dim32(0), C = X.dim32(3), H = X.dim32(1), W = X.dim32(2);\n auto* input_data = X.template data<T_IN>();\n auto* output_data = Y->template mutable_data<T_OUT>();\n\n transform_kernel<\n T_IN, T_OUT><<<N, dim3(16, 16), 0, context->cuda_stream()>>>(\n C, H, W, mean.template data<float>(), std.template data<float>(),\n input_data, output_data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n};\n\ntemplate bool TransformOnGPU<uint8_t, float, CUDAContext>(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n CUDAContext* context);\n\ntemplate bool TransformOnGPU<uint8_t, at::Half, CUDAContext>(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n CUDAContext* context);\n\n} // namespace caffe2\n\n\n###",340 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/image/transform_gpu.h\"\n#include \"caffe2/utils/conversions.h\"\n\n/**\n *\n * Copyright (c) 2016, NVIDIA CORPORATION, All rights reserved\n * Distributed under 2-clause BSD license; see accompanying LICENSE file\n *\n **/\n\nnamespace caffe2 {\n\nnamespace {\n\n// input in (int8, NHWC), output in (fp32, NCHW)\ntemplate <typename In, typename Out>\n__global__ void transform_kernel(\n const int C,\n const int H,\n const int W,\n const float* mean,\n const float* std,\n const In* in,\n Out* out) {\n const auto n = blockIdx.x;\n\n const auto nStride = C*H*W;\n\n // pointers to data for this image\n const In *const input_ptr = &in[n*nStride];\n Out *const output_ptr = &out[n*nStride];\n\n // either read or write uncoalesced - try reading\n for (int c=0; c < C; ++c) {\n for (int h=threadIdx.y; h < H; h += blockDim.y) {\n for (int w=threadIdx.x; w < W; w += blockDim.x) {\n const int in_idx = c + C*w + C*W*h; // HWC\n const int out_idx = c*H*W + h*W + w; // CHW\n\n output_ptr[out_idx] = convert::To<float,Out>(\n (convert::To<In,float>(input_ptr[in_idx])-mean[c]) * std[c]);\n }\n }\n }\n}\n\n}\n\ntemplate <typename T_IN, typename T_OUT, class Context>\n\nbool TransformOnGPU(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n Context* context) {\n const int N = X.dim32(0), C = X.dim32(3), H = X.dim32(1), W = X.dim32(2);\n auto* input_data = X.template data<T_IN>();\n auto* output_data = Y->template mutable_data<T_OUT>();\n\n hipLaunchKernelGGL(( transform_kernel<\n T_IN, T_OUT>), dim3(N), dim3(dim3(16, 16)), 0, context->hip_stream(), \n C, H, W, mean.template data<float>(), std.template data<float>(),\n input_data, output_data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n};\n\ntemplate bool TransformOnGPU<uint8_t, float, HIPContext>(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n HIPContext* context);\n\ntemplate bool TransformOnGPU<uint8_t, at::Half, HIPContext>(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n HIPContext* context);\n\n} // namespace caffe2\n###"341 },342 {343 "cuda": "\n#ifndef CAFFE2_IMAGE_TRANSFORM_GPU_H_\n#define CAFFE2_IMAGE_TRANSFORM_GPU_H_\n\n/**\n *\n * Copyright (c) 2016, NVIDIA CORPORATION, All rights reserved\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *\n * 1. Redistributions of source code must retain the above copyright notice, this\n * list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright notice,\n * this list of conditions and the following disclaimer in the documentation\n * and/or other materials provided with the distribution.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\n * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\n * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\n * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n **/\n\n#include \"caffe2/core/context.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T_IN, typename T_OUT, class Context>\nbool TransformOnGPU(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n Context* context);\n\n} // namespace caffe2\n\n#endif\n\n\n###",344 "hip": " // !!! This is a file automatically generated by hipify!!!\n#ifndef CAFFE2_IMAGE_TRANSFORM_GPU_H_\n#define CAFFE2_IMAGE_TRANSFORM_GPU_H_\n\n/**\n *\n * Copyright (c) 2016, NVIDIA CORPORATION, All rights reserved\n *\n * Redistribution and use in source and binary forms, with or without\n * modification, are permitted provided that the following conditions are met:\n *\n * 1. Redistributions of source code must retain the above copyright notice, this\n * list of conditions and the following disclaimer.\n * 2. Redistributions in binary form must reproduce the above copyright notice,\n * this list of conditions and the following disclaimer in the documentation\n * and/or other materials provided with the distribution.\n *\n * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS \"AS IS\" AND\n * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED\n * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE\n * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR\n * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES\n * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;\n * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND\n * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT\n * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS\n * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\n **/\n\n#include \"caffe2/core/context.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T_IN, typename T_OUT, class Context>\nbool TransformOnGPU(\n Tensor& X,\n Tensor* Y,\n Tensor& mean,\n Tensor& std,\n Context* context);\n\n} // namespace caffe2\n\n#endif\n###"345 },346 {347 "cuda": "\n#include \"caffe2/operators/abs_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nAbsGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(X + i) == T(0)\n ? T(0)\n : (__ldg(X + i) > T(0) ? __ldg(dY + i) : -__ldg(dY + i));\n#else\n dX[i] = X[i] == T(0) ? T(0) : (X[i] > T(0) ? dY[i] : -dY[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AbsGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n AbsGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Abs,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AbsFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n AbsGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AbsGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",348 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/abs_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nAbsGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(X + i) == T(0)\n ? T(0)\n : (__ldg(X + i) > T(0) ? __ldg(dY + i) : -__ldg(dY + i));\n#else\n dX[i] = X[i] == T(0) ? T(0) : (X[i] > T(0) ? dY[i] : -dY[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AbsGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( AbsGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Abs,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AbsFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n AbsGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AbsGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"349 },350 {351 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/accumulate_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(Accumulate, AccumulateOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",352 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/accumulate_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(Accumulate, AccumulateOp<float, HIPContext>);\n} // namespace caffe2\n###"353 },354 {355 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardshrink_kernel(TensorIteratorBase& iter, const Scalar& value) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"hardshrink_cuda\",\n [&]() {\n auto lambd = value.to<scalar_t>();\n gpu_kernel(iter, [lambd] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return (a >= -lambd && a <= lambd) ? scalar_t(0) : a;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(hardshrink_stub, &hardshrink_kernel);\n\n} // namespace at::native\n\n\n###",356 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardshrink_kernel(TensorIteratorBase& iter, const Scalar& value) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"hardshrink_hip\",\n [&]() {\n auto lambd = value.to<scalar_t>();\n gpu_kernel(iter, [lambd] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return (a >= -lambd && a <= lambd) ? scalar_t(0) : a;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(hardshrink_stub, &hardshrink_kernel);\n\n} // namespace at::native\n###"357 },358 {359 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/accuracy_op.h\"\n#include \"caffe2/utils/GpuAtomics.cuh\"\n#include \"caffe2/utils/math.h\"\n\n#include \"caffe2/utils/cub_namespace.cuh\"\n#include <cub/block/block_reduce.cuh>\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void AccuracyKernel(\n const int N,\n const int D,\n const int top_k,\n const float* Xdata,\n const int* labelData,\n float* accuracy) {\n typedef cub::BlockReduce<int, CAFFE_CUDA_NUM_THREADS> BlockReduce;\n __shared__ typename BlockReduce::TempStorage temp_storage;\n int correct = 0;\n for (int row = blockIdx.x; row < N; row += gridDim.x) {\n const int label = labelData[row];\n const float label_pred = Xdata[row * D + label];\n int ngt = 0;\n for (int col = threadIdx.x; col < D; col += blockDim.x) {\n const float pred = Xdata[row * D + col];\n if (pred > label_pred || (pred == label_pred && col <= label)) {\n ++ngt;\n }\n }\n ngt = BlockReduce(temp_storage).Sum(ngt);\n if (ngt <= top_k) {\n ++correct;\n }\n __syncthreads();\n }\n if (threadIdx.x == 0) {\n gpu_atomic_add(accuracy, static_cast<float>(correct));\n }\n}\n\n__global__ void AccuracyDivideKernel(const int N, float* accuracy) {\n *accuracy /= N;\n}\n} // namespace\n\ntemplate <>\nbool AccuracyOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(PREDICTION);\n auto& label = Input(LABEL);\n\n CAFFE_ENFORCE_EQ(X.dim(), 2);\n int N = X.dim32(0);\n int D = X.dim32(1);\n CAFFE_ENFORCE_EQ(label.dim(), 1);\n CAFFE_ENFORCE_EQ(label.dim32(0), N);\n auto* Y = Output(0, vector<int64_t>(), at::dtype<float>());\n float* Ydata = Y->template mutable_data<float>();\n math::Set<float, CUDAContext>(1, 0, Ydata, &context_);\n AccuracyKernel<<<\n std::min(CAFFE_MAXIMUM_NUM_BLOCKS, N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n N, D, top_k_, X.data<float>(), label.data<int>(), Ydata);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n // This is going to be executed only in one single kernel. Not very beautiful,\n // but probably we have to do this?\n AccuracyDivideKernel<<<1, 1, 0, context_.cuda_stream()>>>(\n N, Ydata);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Accuracy, AccuracyOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",360 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/accuracy_op.h\"\n#include \"caffe2/utils/hip/GpuAtomics.cuh\"\n#include \"caffe2/utils/math.h\"\n\n#include \"caffe2/utils/cub_namespace.cuh\"\n#include <hipcub/hipcub.hpp>\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void AccuracyKernel(\n const int N,\n const int D,\n const int top_k,\n const float* Xdata,\n const int* labelData,\n float* accuracy) {\n typedef hipcub::BlockReduce<int, CAFFE_HIP_NUM_THREADS> BlockReduce;\n __shared__ typename BlockReduce::TempStorage temp_storage;\n int correct = 0;\n for (int row = blockIdx.x; row < N; row += gridDim.x) {\n const int label = labelData[row];\n const float label_pred = Xdata[row * D + label];\n int ngt = 0;\n for (int col = threadIdx.x; col < D; col += blockDim.x) {\n const float pred = Xdata[row * D + col];\n if (pred > label_pred || (pred == label_pred && col <= label)) {\n ++ngt;\n }\n }\n ngt = BlockReduce(temp_storage).Sum(ngt);\n if (ngt <= top_k) {\n ++correct;\n }\n __syncthreads();\n }\n if (threadIdx.x == 0) {\n gpu_atomic_add(accuracy, static_cast<float>(correct));\n }\n}\n\n__global__ void AccuracyDivideKernel(const int N, float* accuracy) {\n *accuracy /= N;\n}\n} // namespace\n\ntemplate <>\nbool AccuracyOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(PREDICTION);\n auto& label = Input(LABEL);\n\n CAFFE_ENFORCE_EQ(X.dim(), 2);\n int N = X.dim32(0);\n int D = X.dim32(1);\n CAFFE_ENFORCE_EQ(label.dim(), 1);\n CAFFE_ENFORCE_EQ(label.dim32(0), N);\n auto* Y = Output(0, vector<int64_t>(), at::dtype<float>());\n float* Ydata = Y->template mutable_data<float>();\n math::Set<float, HIPContext>(1, 0, Ydata, &context_);\n hipLaunchKernelGGL(( AccuracyKernel), \n dim3(::min(CAFFE_MAXIMUM_NUM_BLOCKS, N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n N, D, top_k_, X.data<float>(), label.data<int>(), Ydata);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n // This is going to be executed only in one single kernel. Not very beautiful,\n // but probably we have to do this?\n hipLaunchKernelGGL(( AccuracyDivideKernel), dim3(1), dim3(1), 0, context_.hip_stream(), \n N, Ydata);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Accuracy, AccuracyOp<float, HIPContext>);\n} // namespace caffe2\n###"361 },362 {363 "cuda": "\n#include \"caffe2/operators/acos_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void AcosGradientCUDAKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = -__ldg(dY + i) * rsqrtf(1.0f - __ldg(X + i) * __ldg(X + i));\n#else\n dX[i] = -dY[i] * rsqrtf(1.0f - X[i] * X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AcosGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n AcosGradientCUDAKernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Acos,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AcosFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n AcosGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AcosGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",364 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/acos_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void AcosGradientHIPKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = -__ldg(dY + i) * rsqrtf(1.0f - __ldg(X + i) * __ldg(X + i));\n#else\n dX[i] = -dY[i] * rsqrtf(1.0f - X[i] * X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AcosGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( AcosGradientHIPKernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Acos,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AcosFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n AcosGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AcosGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"365 },366 {367 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/alias_with_name.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(AliasWithName, AliasWithNameOp<CUDAContext>);\n\n} // namespace caffe2\n\nC10_EXPORT_CAFFE2_OP_TO_C10_CUDA(\n AliasWithName,\n caffe2::AliasWithNameOp<caffe2::CUDAContext>);\n\n\n###",368 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/alias_with_name.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(AliasWithName, AliasWithNameOp<HIPContext>);\n\n} // namespace caffe2\n\nC10_EXPORT_CAFFE2_OP_TO_C10_HIP(\n AliasWithName,\n caffe2::AliasWithNameOp<caffe2::HIPContext>);\n###"369 },370 {371 "cuda": "\n#include \"caffe2/operators/asin_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void AsinGradientCUDAKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * rsqrtf(1.0f - __ldg(X + i) * __ldg(X + i));\n#else\n dX[i] = dY[i] * rsqrtf(1.0f - X[i] * X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AsinGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n AsinGradientCUDAKernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Asin,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AsinFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n AsinGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AsinGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",372 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/asin_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void AsinGradientHIPKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * rsqrtf(1.0f - __ldg(X + i) * __ldg(X + i));\n#else\n dX[i] = dY[i] * rsqrtf(1.0f - X[i] * X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AsinGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( AsinGradientHIPKernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Asin,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AsinFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n AsinGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AsinGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"373 },374 {375 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/assert_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Assert, AssertOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",376 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/assert_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Assert, AssertOp<HIPContext>);\n\n} // namespace caffe2\n###"377 },378 {379 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/async_net_barrier_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(AsyncNetBarrier, AsyncNetBarrierOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",380 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/async_net_barrier_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(AsyncNetBarrier, AsyncNetBarrierOp<HIPContext>);\n\n} // namespace caffe2\n###"381 },382 {383 "cuda": "\n#include \"caffe2/operators/atan_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nAtanGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / (T(1) + __ldg(X + i) * __ldg(X + i));\n#else\n dX[i] = dY[i] / (T(1) + X[i] * X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AtanGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n AtanGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Atan,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AtanFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n AtanGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n AtanGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",384 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/atan_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nAtanGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / (T(1) + __ldg(X + i) * __ldg(X + i));\n#else\n dX[i] = dY[i] / (T(1) + X[i] * X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool AtanGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( AtanGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Atan,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AtanFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n AtanGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n AtanGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"385 },386 {387 "cuda": "\n#include \"caffe2/operators/batch_matmul_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool BatchMatMulOp<CUDAContext, DefaultEngine>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\nREGISTER_CUDA_OPERATOR(BatchMatMul, BatchMatMulOp<CUDAContext>);\n\n\n#if !defined(USE_ROCM)\n\ntemplate <>\nbool BatchMatMulOp<CUDAContext, TensorCoreEngine>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\nREGISTER_CUDA_OPERATOR_WITH_ENGINE(\n BatchMatMul,\n TENSORCORE,\n BatchMatMulOp<CUDAContext, TensorCoreEngine>);\n\n#endif\n\n} // namespace caffe2\n\n\n###",388 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/batch_matmul_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool BatchMatMulOp<HIPContext, DefaultEngine>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\nREGISTER_HIP_OPERATOR(BatchMatMul, BatchMatMulOp<HIPContext>);\n\n\n#if !defined(USE_ROCM)\n\ntemplate <>\nbool BatchMatMulOp<HIPContext, TensorCoreEngine>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\nREGISTER_HIP_OPERATOR_WITH_ENGINE(\n BatchMatMul,\n TENSORCORE,\n BatchMatMulOp<HIPContext, TensorCoreEngine>);\n\n#endif\n\n} // namespace caffe2\n###"389 },390 {391 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/bucketize_op.h\"\n\n#include <thrust/binary_search.h>\n#include <thrust/device_vector.h>\n\nnamespace caffe2 {\n\n__global__ void BucketizeOpKernel(\n const int N,\n const int M,\n const float* bounds,\n const float* X,\n int32_t* out) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n int32_t low = -1, high = M;\n while (high - low > 1) {\n const int32_t median = low + (high - low) / 2;\n if (bounds[median] < X[i]) {\n low = median;\n } else {\n high = median;\n }\n }\n out[i] = high;\n }\n}\n\ntemplate <>\nbool BucketizeOp<CUDAContext>::RunOnDevice() {\n auto& input = Input(X);\n CAFFE_ENFORCE_GE(input.dim(), 1);\n\n auto N = input.numel();\n auto* output = Output(INDICES, input.sizes(), at::dtype<int32_t>());\n const auto* input_data = input.template data<float>();\n auto* output_data = output->template mutable_data<int32_t>();\n\n BucketizeOpKernel<<<\n CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n N,\n boundaries_device_.numel(),\n boundaries_device_.data<float>(),\n input_data,\n output_data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n};\n\nREGISTER_CUDA_OPERATOR(Bucketize, BucketizeOp<CUDAContext>);\n} // namespace caffe2\n\nusing BucketizeCUDA = caffe2::BucketizeOp<caffe2::CUDAContext>;\n\nC10_EXPORT_CAFFE2_OP_TO_C10_CUDA(\n Bucketize,\n BucketizeCUDA);\n\n\n###",392 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/bucketize_op.h\"\n\n#include <thrust/binary_search.h>\n#include <thrust/device_vector.h>\n\nnamespace caffe2 {\n\n__global__ void BucketizeOpKernel(\n const int N,\n const int M,\n const float* bounds,\n const float* X,\n int32_t* out) {\n HIP_1D_KERNEL_LOOP(i, N) {\n int32_t low = -1, high = M;\n while (high - low > 1) {\n const int32_t median = low + (high - low) / 2;\n if (bounds[median] < X[i]) {\n low = median;\n } else {\n high = median;\n }\n }\n out[i] = high;\n }\n}\n\ntemplate <>\nbool BucketizeOp<HIPContext>::RunOnDevice() {\n auto& input = Input(X);\n CAFFE_ENFORCE_GE(input.dim(), 1);\n\n auto N = input.numel();\n auto* output = Output(INDICES, input.sizes(), at::dtype<int32_t>());\n const auto* input_data = input.template data<float>();\n auto* output_data = output->template mutable_data<int32_t>();\n\n hipLaunchKernelGGL(( BucketizeOpKernel), \n dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n N,\n boundaries_device_.numel(),\n boundaries_device_.data<float>(),\n input_data,\n output_data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n};\n\nREGISTER_HIP_OPERATOR(Bucketize, BucketizeOp<HIPContext>);\n} // namespace caffe2\n\nusing BucketizeHIP = caffe2::BucketizeOp<caffe2::HIPContext>;\n\nC10_EXPORT_CAFFE2_OP_TO_C10_HIP(\n Bucketize,\n BucketizeHIP);\n###"393 },394 {395 "cuda": "\n#include \"caffe2/operators/cbrt_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nCbrtGradientCUDAKernel(const int N, const T* dY, const T* Y, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / (__ldg(Y + i) * __ldg(Y + i) * T(3));\n#else\n dX[i] = dY[i] / (Y[i] * Y[i] * T(3));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CbrtGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& dY_dims,\n const std::vector<int>& /* Y_dims */,\n const T* dY,\n const T* Y,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n dY_dims.cbegin(), dY_dims.cend(), 1, std::multiplies<int>());\n CbrtGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Cbrt,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n CbrtFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n CbrtGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n CbrtGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",396 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/cbrt_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nCbrtGradientHIPKernel(const int N, const T* dY, const T* Y, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / (__ldg(Y + i) * __ldg(Y + i) * T(3));\n#else\n dX[i] = dY[i] / (Y[i] * Y[i] * T(3));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CbrtGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& dY_dims,\n const std::vector<int>& /* Y_dims */,\n const T* dY,\n const T* Y,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n dY_dims.cbegin(), dY_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( CbrtGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Cbrt,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n CbrtFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n CbrtGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n CbrtGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"397 },398 {399 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardsigmoid_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"hardsigmoid_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t one_sixth(1.0f / 6.0f);\n const opmath_t three(3.0f);\n const opmath_t six(6.0f);\n gpu_kernel(\n iter,\n [zero, one_sixth, three, six] GPU_LAMBDA(\n scalar_t self_val) -> scalar_t {\n opmath_t x = static_cast<opmath_t>(self_val);\n return std::min(std::max(x + three, zero), six) * one_sixth;\n });\n });\n}\n\nvoid hardsigmoid_backward_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"hardsigmoid_backward_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t three(3.0f);\n const opmath_t neg_three(-3.0f);\n const opmath_t one_sixth(1.0f / 6.0f);\n gpu_kernel(\n iter,\n [zero, three, neg_three, one_sixth] GPU_LAMBDA(\n scalar_t grad_val_, scalar_t self_val_) -> scalar_t {\n opmath_t grad_val = static_cast<opmath_t>(grad_val_);\n opmath_t self_val = static_cast<opmath_t>(self_val_);\n return (self_val > neg_three && self_val < three)\n ? grad_val * one_sixth\n : zero;\n });\n });\n}\n\n} // namespace\n\nREGISTER_DISPATCH(hardsigmoid_stub, &hardsigmoid_kernel);\nREGISTER_DISPATCH(hardsigmoid_backward_stub, &hardsigmoid_backward_kernel);\n\n} // namespace at::native\n\n\n###",400 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardsigmoid_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"hardsigmoid_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t one_sixth(1.0f / 6.0f);\n const opmath_t three(3.0f);\n const opmath_t six(6.0f);\n gpu_kernel(\n iter,\n [zero, one_sixth, three, six] GPU_LAMBDA(\n scalar_t self_val) -> scalar_t {\n opmath_t x = static_cast<opmath_t>(self_val);\n return ::min(::max(x + three, zero), six) * one_sixth;\n });\n });\n}\n\nvoid hardsigmoid_backward_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"hardsigmoid_backward_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t three(3.0f);\n const opmath_t neg_three(-3.0f);\n const opmath_t one_sixth(1.0f / 6.0f);\n gpu_kernel(\n iter,\n [zero, three, neg_three, one_sixth] GPU_LAMBDA(\n scalar_t grad_val_, scalar_t self_val_) -> scalar_t {\n opmath_t grad_val = static_cast<opmath_t>(grad_val_);\n opmath_t self_val = static_cast<opmath_t>(self_val_);\n return (self_val > neg_three && self_val < three)\n ? grad_val * one_sixth\n : zero;\n });\n });\n}\n\n} // namespace\n\nREGISTER_DISPATCH(hardsigmoid_stub, &hardsigmoid_kernel);\nREGISTER_DISPATCH(hardsigmoid_backward_stub, &hardsigmoid_backward_kernel);\n\n} // namespace at::native\n###"401 },402 {403 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/ceil_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\n__global__ void CeilKernel(const int N, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = std::ceil(X[i]);\n }\n}\n\ntemplate <>\nbool CeilOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n CeilKernel<<<\n CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Ceil, CeilOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",404 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/ceil_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\n__global__ void CeilKernel(const int N, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = ::ceil(X[i]);\n }\n}\n\ntemplate <>\nbool CeilOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( CeilKernel), \n dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Ceil, CeilOp<float, HIPContext>);\n} // namespace caffe2\n###"405 },406 {407 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/clip_op.h\"\n\nnamespace caffe2 {\nnamespace {\n\ntemplate <typename T>\n__device__ T cuda_min(T x, T y);\ntemplate <typename T>\n__device__ T cuda_max(T x, T y);\ntemplate <>\n__device__ float cuda_min(float x, float y) { return fminf(x, y); }\ntemplate <>\n__device__ float cuda_max(float x, float y) { return fmaxf(x, y); }\n\n// Disabled since we don't use it right now.\n/*\ntemplate <>\n__device__ double cuda_min(double x, double y) { return fmin(x, y); }\ntemplate <>\n__device__ double cuda_max(double x, double y) { return fmax(x, y); }\n*/\n\n\ntemplate <typename T>\n__global__ void ClipKernel(const int N, const T minval, const T maxval,\n const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = cuda_min<T>(cuda_max<T>(X[i], minval), maxval);\n }\n}\n\ntemplate <typename T>\n__global__ void ClipGradientKernel(const int N, const T minval,\n const T maxval, const T* Y,\n const T* dY, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n dX[i] = dY[i] * (Y[i] > minval && Y[i] < maxval);\n }\n}\n} // namespace\n\ntemplate <>\nbool ClipOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GE(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n ClipKernel<<<\n CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(), min_, max_, X.data<float>(), Y->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool ClipGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n CAFFE_ENFORCE_GE(Y.numel(), 0);\n CAFFE_ENFORCE_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n ClipGradientKernel<<<\n CAFFE_GET_BLOCKS(Y.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n Y.numel(),\n min_,\n max_,\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Clip, ClipOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(ClipGradient, ClipGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",408 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/clip_op.h\"\n\nnamespace caffe2 {\nnamespace {\n\ntemplate <typename T>\n__device__ T hip_min(T x, T y);\ntemplate <typename T>\n__device__ T hip_max(T x, T y);\ntemplate <>\n__device__ float hip_min(float x, float y) { return fminf(x, y); }\ntemplate <>\n__device__ float hip_max(float x, float y) { return fmaxf(x, y); }\n\n// Disabled since we don't use it right now.\n/*\ntemplate <>\n__device__ double hip_min(double x, double y) { return fmin(x, y); }\ntemplate <>\n__device__ double hip_max(double x, double y) { return fmax(x, y); }\n*/\n\n\ntemplate <typename T>\n__global__ void ClipKernel(const int N, const T minval, const T maxval,\n const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = hip_min<T>(hip_max<T>(X[i], minval), maxval);\n }\n}\n\ntemplate <typename T>\n__global__ void ClipGradientKernel(const int N, const T minval,\n const T maxval, const T* Y,\n const T* dY, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n dX[i] = dY[i] * (Y[i] > minval && Y[i] < maxval);\n }\n}\n} // namespace\n\ntemplate <>\nbool ClipOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GE(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( ClipKernel), \n dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(), min_, max_, X.data<float>(), Y->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool ClipGradientOp<float, HIPContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n CAFFE_ENFORCE_GE(Y.numel(), 0);\n CAFFE_ENFORCE_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( ClipGradientKernel), \n dim3(CAFFE_GET_BLOCKS(Y.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n Y.numel(),\n min_,\n max_,\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Clip, ClipOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(ClipGradient, ClipGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"409 },410 {411 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/operators/no_default_engine_op.h\"\n\nnamespace caffe2 {\n// Communication operators do not have default engines.\nREGISTER_CUDA_OPERATOR(CreateCommonWorld, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(CloneCommonWorld, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Broadcast, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Reduce, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Allgather, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Allreduce, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(SendTensor, NoDefaultEngineOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(ReceiveTensor, NoDefaultEngineOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",412 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/operators/no_default_engine_op.h\"\n\nnamespace caffe2 {\n// Communication operators do not have default engines.\nREGISTER_HIP_OPERATOR(CreateCommonWorld, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(CloneCommonWorld, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Broadcast, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Reduce, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Allgather, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Allreduce, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(SendTensor, NoDefaultEngineOp<HIPContext>);\nREGISTER_HIP_OPERATOR(ReceiveTensor, NoDefaultEngineOp<HIPContext>);\n\n} // namespace caffe2\n###"413 },414 {415 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/concat_split_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(Split, SplitOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Concat, ConcatOp<CUDAContext>);\n\n// Backward compatibility settings\nREGISTER_CUDA_OPERATOR(DepthSplit, SplitOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(DepthConcat, ConcatOp<CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(SplitByLengths, SplitByLengthsOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",416 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/concat_split_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(Split, SplitOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Concat, ConcatOp<HIPContext>);\n\n// Backward compatibility settings\nREGISTER_HIP_OPERATOR(DepthSplit, SplitOp<HIPContext>);\nREGISTER_HIP_OPERATOR(DepthConcat, ConcatOp<HIPContext>);\n\nREGISTER_HIP_OPERATOR(SplitByLengths, SplitByLengthsOp<HIPContext>);\n} // namespace caffe2\n###"417 },418 {419 "cuda": "\n#include \"caffe2/operators/conv_op.h\"\n#include \"caffe2/operators/conv_op_impl.h\"\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(Conv, ConvOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(ConvGradient, ConvGradientOp<float, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(Conv1D, ConvOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(Conv1DGradient, ConvGradientOp<float, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(Conv2D, ConvOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(Conv2DGradient, ConvGradientOp<float, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(Conv3D, ConvOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(Conv3DGradient, ConvGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",420 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/conv_op.h\"\n#include \"caffe2/operators/conv_op_impl.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(Conv, ConvOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(ConvGradient, ConvGradientOp<float, HIPContext>);\n\nREGISTER_HIP_OPERATOR(Conv1D, ConvOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(Conv1DGradient, ConvGradientOp<float, HIPContext>);\n\nREGISTER_HIP_OPERATOR(Conv2D, ConvOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(Conv2DGradient, ConvGradientOp<float, HIPContext>);\n\nREGISTER_HIP_OPERATOR(Conv3D, ConvOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(Conv3DGradient, ConvGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"421 },422 {423 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/conv_op_shared.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nvoid createSharedBuffer<CUDAContext>(Workspace* ws) {\n auto* mutexPtr = ws->CreateBlob(\"__CAFFE2_SHARED_CONV_BUFFER_CUDA_MUTEX__\")\n ->GetMutable<std::unique_ptr<std::mutex>>();\n mutexPtr->reset(new std::mutex());\n ws->CreateBlob(\"__CAFFE2_SHARED_CONV_BUFFER_CUDA__\");\n}\n\ntemplate <>\nvoid runWithSharedBuffer<CUDAContext>(\n Workspace* ws,\n std::function<void(Tensor* buffer)> f) {\n auto* mutexBlob = ws->GetBlob(\"__CAFFE2_SHARED_CONV_BUFFER_CUDA_MUTEX__\");\n CAFFE_ENFORCE(mutexBlob, \"Must call createSharedBuffer() first\");\n\n auto* mutexPtr = mutexBlob->GetMutable<std::unique_ptr<std::mutex>>();\n std::lock_guard<std::mutex> g(**mutexPtr);\n auto* buffer = BlobGetMutableTensor(\n ws->GetBlob(\"__CAFFE2_SHARED_CONV_BUFFER_CUDA__\"), CUDA);\n f(buffer);\n}\n}\n\n\n###",424 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/conv_op_shared.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nvoid createSharedBuffer<HIPContext>(Workspace* ws) {\n auto* mutexPtr = ws->CreateBlob(\"__CAFFE2_SHARED_CONV_BUFFER_HIP_MUTEX__\")\n ->GetMutable<std::unique_ptr<std::mutex>>();\n mutexPtr->reset(new std::mutex());\n ws->CreateBlob(\"__CAFFE2_SHARED_CONV_BUFFER_HIP__\");\n}\n\ntemplate <>\nvoid runWithSharedBuffer<HIPContext>(\n Workspace* ws,\n std::function<void(Tensor* buffer)> f) {\n auto* mutexBlob = ws->GetBlob(\"__CAFFE2_SHARED_CONV_BUFFER_HIP_MUTEX__\");\n CAFFE_ENFORCE(mutexBlob, \"Must call createSharedBuffer() first\");\n\n auto* mutexPtr = mutexBlob->GetMutable<std::unique_ptr<std::mutex>>();\n std::lock_guard<std::mutex> g(**mutexPtr);\n auto* buffer = BlobGetMutableTensor(\n ws->GetBlob(\"__CAFFE2_SHARED_CONV_BUFFER_HIP__\"), HIP);\n f(buffer);\n}\n}\n###"425 },426 {427 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/conv_transpose_op.h\"\n#include \"caffe2/operators/conv_transpose_op_impl.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(ConvTranspose, ConvTransposeOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n ConvTransposeGradient,\n ConvTransposeGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",428 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/conv_transpose_op.h\"\n#include \"caffe2/operators/conv_transpose_op_impl.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(ConvTranspose, ConvTransposeOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n ConvTransposeGradient,\n ConvTransposeGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"429 },430 {431 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/copy_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nclass CopyOnDeviceLikeOp<CUDAContext, CUDAContext, CUDAContext>\n : public Operator<CUDAContext> {\n public:\n template <class... Args>\n explicit CopyOnDeviceLikeOp(Args&&... args)\n : Operator<CUDAContext>(std::forward<Args>(args)...) {}\n USE_OPERATOR_FUNCTIONS(CUDAContext);\n\n bool RunOnDevice() override {\n auto& input = Input(0);\n auto* output = OperatorBase::Output<Tensor>(0, CUDA);\n CUDAContext context(GetGPUIDForPointer(Input(1).raw_data()));\n output->ResizeLike(input);\n context.template CopyItems<CUDAContext, CUDAContext>(\n input.meta(),\n input.numel(),\n input.raw_data(),\n output->raw_mutable_data(input.meta()));\n return true;\n }\n};\n\n// From CPU, copy it to whatever the current context\nREGISTER_CUDA_OPERATOR(\n CopyFromCPUInput,\n CopyOp<CUDAContext, CUDAContext, CPUContext>);\n\n// CopyGPUToCPU and CopyCPUToGPU should both be carried out in a cuda context,\n// since gpu code will be involved.\nREGISTER_CUDA_OPERATOR(\n CopyGPUToCPU,\n CopyOp<CUDAContext, CPUContext, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n CopyCPUToGPU,\n CopyOp<CUDAContext, CUDAContext, CPUContext>);\n// If we only specify Copy, we assume that it is a gpu to gpu copy - maybe\n// involving different GPUs.\nREGISTER_CUDA_OPERATOR(Copy, CopyOp<CUDAContext, CUDAContext, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(\n CopyOnDeviceLike,\n CopyOnDeviceLikeOp<CUDAContext, CUDAContext, CUDAContext>);\n} // namespace caffe2\n\nusing CopyGPUToCPU_CUDA = caffe2::\n CopyOp<caffe2::CUDAContext, caffe2::CPUContext, caffe2::CUDAContext>;\nusing CopyCPUToGPU_CUDA = caffe2::\n CopyOp<caffe2::CUDAContext, caffe2::CUDAContext, caffe2::CPUContext>;\n\nC10_EXPORT_CAFFE2_OP_TO_C10_CUDA(CopyGPUToCPU, CopyGPUToCPU_CUDA);\n\nC10_EXPORT_CAFFE2_OP_TO_C10_CPU_KERNEL_ONLY(CopyCPUToGPU, CopyCPUToGPU_CUDA);\n\n\n###",432 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/copy_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nclass CopyOnDeviceLikeOp<HIPContext, HIPContext, HIPContext>\n : public Operator<HIPContext> {\n public:\n template <class... Args>\n explicit CopyOnDeviceLikeOp(Args&&... args)\n : Operator<HIPContext>(std::forward<Args>(args)...) {}\n USE_OPERATOR_FUNCTIONS(HIPContext);\n\n bool RunOnDevice() override {\n auto& input = Input(0);\n auto* output = OperatorBase::Output<Tensor>(0, HIP);\n HIPContext context(GetGPUIDForPointer(Input(1).raw_data()));\n output->ResizeLike(input);\n context.template CopyItems<HIPContext, HIPContext>(\n input.meta(),\n input.numel(),\n input.raw_data(),\n output->raw_mutable_data(input.meta()));\n return true;\n }\n};\n\n// From CPU, copy it to whatever the current context\nREGISTER_HIP_OPERATOR(\n CopyFromCPUInput,\n CopyOp<HIPContext, HIPContext, CPUContext>);\n\n// CopyGPUToCPU and CopyCPUToGPU should both be carried out in a cuda context,\n// since gpu code will be involved.\nREGISTER_HIP_OPERATOR(\n CopyGPUToCPU,\n CopyOp<HIPContext, CPUContext, HIPContext>);\nREGISTER_HIP_OPERATOR(\n CopyCPUToGPU,\n CopyOp<HIPContext, HIPContext, CPUContext>);\n// If we only specify Copy, we assume that it is a gpu to gpu copy - maybe\n// involving different GPUs.\nREGISTER_HIP_OPERATOR(Copy, CopyOp<HIPContext, HIPContext, HIPContext>);\n\nREGISTER_HIP_OPERATOR(\n CopyOnDeviceLike,\n CopyOnDeviceLikeOp<HIPContext, HIPContext, HIPContext>);\n} // namespace caffe2\n\nusing CopyGPUToCPU_HIP = caffe2::\n CopyOp<caffe2::HIPContext, caffe2::CPUContext, caffe2::HIPContext>;\nusing CopyCPUToGPU_HIP = caffe2::\n CopyOp<caffe2::HIPContext, caffe2::HIPContext, caffe2::CPUContext>;\n\nC10_EXPORT_CAFFE2_OP_TO_C10_HIP(CopyGPUToCPU, CopyGPUToCPU_HIP);\n\nC10_EXPORT_CAFFE2_OP_TO_C10_CPU_KERNEL_ONLY(CopyCPUToGPU, CopyCPUToGPU_HIP);\n###"433 },434 {435 "cuda": "\n#include \"caffe2/operators/cosh_op.h\"\n\n#include <c10/util/accumulate.h>\n#include \"caffe2/core/context_gpu.h\"\n\n#include <algorithm>\n#include <functional>\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void CoshGradientCUDAKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * sinhf(__ldg(X + i));\n#else\n dX[i] = dY[i] * sinhf(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CoshGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& /* dY_dims */,\n const std::vector<int>& X_dims,\n const T* dY,\n const T* X,\n T* dX,\n CUDAContext* context) const {\n const auto size = c10::multiply_integers(X_dims.cbegin(), X_dims.cend());\n CoshGradientCUDAKernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Cosh,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n CoshFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n CoshGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n CoshGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",436 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/cosh_op.h\"\n\n#include <c10/util/accumulate.h>\n#include \"caffe2/core/hip/context_gpu.h\"\n\n#include <algorithm>\n#include <functional>\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void CoshGradientHIPKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * sinhf(__ldg(X + i));\n#else\n dX[i] = dY[i] * sinhf(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CoshGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& /* dY_dims */,\n const std::vector<int>& X_dims,\n const T* dY,\n const T* X,\n T* dX,\n HIPContext* context) const {\n const auto size = c10::multiply_integers(X_dims.cbegin(), X_dims.cend());\n hipLaunchKernelGGL(( CoshGradientHIPKernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Cosh,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n CoshFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n CoshGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n CoshGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"437 },438 {439 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/cosine_embedding_criterion_op.h\"\n\nnamespace caffe2 {\nnamespace {\n\n\n__global__ void CECKernel(\n const int N, const float* S, const int* Y, const float margin,\n float* output) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n output[i] = Y[i] == 1 ? (1. - S[i]) : fmaxf(0.f, S[i] - margin);\n }\n}\n\n__global__ void CECGradientKernel(\n const int N, const float* S, const int* Y, const float* dOutput,\n const float margin, float* dS) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n dS[i] = dOutput[i] * (Y[i] == 1 ? -1 : static_cast<float>(S[i] >= margin));\n }\n}\n} // namespace\n\ntemplate <>\nbool CosineEmbeddingCriterionOp<CUDAContext>::RunOnDevice() {\n auto& S = Input(0);\n auto& Y = Input(1);\n\n CAFFE_ENFORCE(S.numel() == Y.numel(),\n \"The embedding and label should have the same size.\");\n auto* output = Output(0, S.sizes(), at::dtype<float>());\n\n const float* Sdata = S.data<float>();\n const int* Ydata = Y.data<int>();\n float* output_data = output->template mutable_data<float>();\n\n CECKernel<<<CAFFE_GET_BLOCKS(S.numel()), CAFFE_CUDA_NUM_THREADS,\n 0, context_.cuda_stream()>>>(\n S.numel(), Sdata, Ydata, margin_, output_data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool CosineEmbeddingCriterionGradientOp<CUDAContext>::RunOnDevice() {\n auto& S = Input(0);\n auto& Y = Input(1);\n auto& dOutput = Input(2);\n\n\n auto* dS = Output(0, S.sizes(), at::dtype<float>());\n\n const float* Sdata = S.data<float>();\n const int* Ydata = Y.data<int>();\n const float* dOutput_data = dOutput.data<float>();\n float* dSdata = dS->template mutable_data<float>();\n CECGradientKernel<<<CAFFE_GET_BLOCKS(S.numel()), CAFFE_CUDA_NUM_THREADS,\n 0, context_.cuda_stream()>>>(\n S.numel(), Sdata, Ydata, dOutput_data, margin_, dSdata);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n CosineEmbeddingCriterion,\n CosineEmbeddingCriterionOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n CosineEmbeddingCriterionGradient,\n CosineEmbeddingCriterionGradientOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",440 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/cosine_embedding_criterion_op.h\"\n\nnamespace caffe2 {\nnamespace {\n\n\n__global__ void CECKernel(\n const int N, const float* S, const int* Y, const float margin,\n float* output) {\n HIP_1D_KERNEL_LOOP(i, N) {\n output[i] = Y[i] == 1 ? (1. - S[i]) : fmaxf(0.f, S[i] - margin);\n }\n}\n\n__global__ void CECGradientKernel(\n const int N, const float* S, const int* Y, const float* dOutput,\n const float margin, float* dS) {\n HIP_1D_KERNEL_LOOP(i, N) {\n dS[i] = dOutput[i] * (Y[i] == 1 ? -1 : static_cast<float>(S[i] >= margin));\n }\n}\n} // namespace\n\ntemplate <>\nbool CosineEmbeddingCriterionOp<HIPContext>::RunOnDevice() {\n auto& S = Input(0);\n auto& Y = Input(1);\n\n CAFFE_ENFORCE(S.numel() == Y.numel(),\n \"The embedding and label should have the same size.\");\n auto* output = Output(0, S.sizes(), at::dtype<float>());\n\n const float* Sdata = S.data<float>();\n const int* Ydata = Y.data<int>();\n float* output_data = output->template mutable_data<float>();\n\n hipLaunchKernelGGL(( CECKernel), dim3(CAFFE_GET_BLOCKS(S.numel())), dim3(CAFFE_HIP_NUM_THREADS),\n 0, context_.hip_stream(), \n S.numel(), Sdata, Ydata, margin_, output_data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool CosineEmbeddingCriterionGradientOp<HIPContext>::RunOnDevice() {\n auto& S = Input(0);\n auto& Y = Input(1);\n auto& dOutput = Input(2);\n\n\n auto* dS = Output(0, S.sizes(), at::dtype<float>());\n\n const float* Sdata = S.data<float>();\n const int* Ydata = Y.data<int>();\n const float* dOutput_data = dOutput.data<float>();\n float* dSdata = dS->template mutable_data<float>();\n hipLaunchKernelGGL(( CECGradientKernel), dim3(CAFFE_GET_BLOCKS(S.numel())), dim3(CAFFE_HIP_NUM_THREADS),\n 0, context_.hip_stream(), \n S.numel(), Sdata, Ydata, dOutput_data, margin_, dSdata);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n CosineEmbeddingCriterion,\n CosineEmbeddingCriterionOp<HIPContext>);\nREGISTER_HIP_OPERATOR(\n CosineEmbeddingCriterionGradient,\n CosineEmbeddingCriterionGradientOp<HIPContext>);\n} // namespace caffe2\n###"441 },442 {443 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardswish_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, iter.dtype(), \"hardswish_cuda\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t one_sixth(1.0f / 6.0f);\n const opmath_t three(3.0f);\n const opmath_t six(6.0f);\n gpu_kernel(iter, [zero, one_sixth, three, six]GPU_LAMBDA(scalar_t self_val) -> scalar_t {\n opmath_t x = static_cast<opmath_t>(self_val);\n return x * std::min(std::max(x + three, zero), six) * one_sixth;\n });\n });\n}\n\nvoid hardswish_backward_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, iter.dtype(), \"hardswish_backward_cuda\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t three(3.0f);\n const opmath_t neg_three(-3.0f);\n const opmath_t one_half(0.5f);\n gpu_kernel(\n iter,\n [zero, three, neg_three, one_half]GPU_LAMBDA(scalar_t grad_val_, scalar_t self_val_) -> scalar_t {\n opmath_t grad_val = static_cast<opmath_t>(grad_val_);\n opmath_t self_val = static_cast<opmath_t>(self_val_);\n if (self_val < neg_three) {\n return zero;\n } else if (self_val <= three) {\n return grad_val * ((self_val / three) + one_half);\n } else {\n return grad_val;\n }\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(hardswish_stub, &hardswish_kernel);\nREGISTER_DISPATCH(hardswish_backward_stub, &hardswish_backward_kernel);\n\n} // namespace at::native\n\n\n###",444 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardswish_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, iter.dtype(), \"hardswish_hip\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t one_sixth(1.0f / 6.0f);\n const opmath_t three(3.0f);\n const opmath_t six(6.0f);\n gpu_kernel(iter, [zero, one_sixth, three, six]GPU_LAMBDA(scalar_t self_val) -> scalar_t {\n opmath_t x = static_cast<opmath_t>(self_val);\n return x * ::min(::max(x + three, zero), six) * one_sixth;\n });\n });\n}\n\nvoid hardswish_backward_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(at::ScalarType::Half, at::ScalarType::BFloat16, iter.dtype(), \"hardswish_backward_hip\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t zero(0.0f);\n const opmath_t three(3.0f);\n const opmath_t neg_three(-3.0f);\n const opmath_t one_half(0.5f);\n gpu_kernel(\n iter,\n [zero, three, neg_three, one_half]GPU_LAMBDA(scalar_t grad_val_, scalar_t self_val_) -> scalar_t {\n opmath_t grad_val = static_cast<opmath_t>(grad_val_);\n opmath_t self_val = static_cast<opmath_t>(self_val_);\n if (self_val < neg_three) {\n return zero;\n } else if (self_val <= three) {\n return grad_val * ((self_val / three) + one_half);\n } else {\n return grad_val;\n }\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(hardswish_stub, &hardswish_kernel);\nREGISTER_DISPATCH(hardswish_backward_stub, &hardswish_backward_kernel);\n\n} // namespace at::native\n###"445 },446 {447 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAConfig.h>\n#include <ATen/cuda/cub.cuh>\n\nnamespace at {\nnamespace cuda {\nnamespace cub {\n\ntemplate <typename key_t>\nvoid radix_sort_keys(\n const key_t* keys_in,\n key_t* keys_out,\n int64_t n,\n bool descending,\n int64_t begin_bit,\n int64_t end_bit) {\n TORCH_CHECK(\n n <= std::numeric_limits<int>::max(),\n \"cub sort does not support sorting more than INT_MAX elements\");\n using key_t_ = typename detail::cuda_type<key_t>::type;\n\n const key_t_* keys_in_ = reinterpret_cast<const key_t_*>(keys_in);\n key_t_* keys_out_ = reinterpret_cast<key_t_*>(keys_out);\n\n if (descending) {\n CUB_WRAPPER(\n NO_ROCM(at_cuda_detail)::cub::DeviceRadixSort::SortKeysDescending,\n keys_in_,\n keys_out_,\n n,\n begin_bit,\n end_bit,\n c10::cuda::getCurrentCUDAStream());\n } else {\n CUB_WRAPPER(\n NO_ROCM(at_cuda_detail)::cub::DeviceRadixSort::SortKeys,\n keys_in_,\n keys_out_,\n n,\n begin_bit,\n end_bit,\n c10::cuda::getCurrentCUDAStream());\n }\n}\n\n#define AT_INSTATIATE_CUB_TEMPLATES(scalar_t, ScalarType) \\\n template void radix_sort_keys( \\\n const scalar_t* keys_in, \\\n scalar_t* keys_out, \\\n int64_t n, \\\n bool descending, \\\n int64_t begin_bit, \\\n int64_t end_bit);\n\nAT_FORALL_SCALAR_TYPES_AND2(Bool, Half, AT_INSTATIATE_CUB_TEMPLATES)\n\n} // namespace cub\n} // namespace cuda\n} // namespace at\n\n\n###",448 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPConfig.h>\n#include <ATen/hip\\cub.cuh>\n\nnamespace at {\nnamespace hip {\nnamespace cub {\n\ntemplate <typename key_t>\nvoid radix_sort_keys(\n const key_t* keys_in,\n key_t* keys_out,\n int64_t n,\n bool descending,\n int64_t begin_bit,\n int64_t end_bit) {\n TORCH_CHECK(\n n <= std::numeric_limits<int>::max(),\n \"cub sort does not support sorting more than INT_MAX elements\");\n using key_t_ = typename detail::hip_type<key_t>::type;\n\n const key_t_* keys_in_ = reinterpret_cast<const key_t_*>(keys_in);\n key_t_* keys_out_ = reinterpret_cast<key_t_*>(keys_out);\n\n if (descending) {\n CUB_WRAPPER(\n NO_ROCM(at_hip_detail)::hipcub::DeviceRadixSort::SortKeysDescending,\n keys_in_,\n keys_out_,\n n,\n begin_bit,\n end_bit,\n c10::hip::getCurrentHIPStream());\n } else {\n CUB_WRAPPER(\n NO_ROCM(at_hip_detail)::hipcub::DeviceRadixSort::SortKeys,\n keys_in_,\n keys_out_,\n n,\n begin_bit,\n end_bit,\n c10::hip::getCurrentHIPStream());\n }\n}\n\n#define AT_INSTATIATE_CUB_TEMPLATES(scalar_t, ScalarType) \\\n template void radix_sort_keys( \\\n const scalar_t* keys_in, \\\n scalar_t* keys_out, \\\n int64_t n, \\\n bool descending, \\\n int64_t begin_bit, \\\n int64_t end_bit);\n\nAT_FORALL_SCALAR_TYPES_AND2(Bool, Half, AT_INSTATIATE_CUB_TEMPLATES)\n\n} // namespace cub\n} // namespace hip\n} // namespace at\n###"449 },450 {451 "cuda": "\n#include \"caffe2/operators/cos_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nCosGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = -__ldg(dY + i) * sin(__ldg(X + i));\n#else\n dX[i] = -dY[i] * sin(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CosGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n CosGradientCUDAKernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Cos,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n CosFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n CosGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n CosGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",452 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/cos_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nCosGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = -__ldg(dY + i) * sin(__ldg(X + i));\n#else\n dX[i] = -dY[i] * sin(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CosGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( CosGradientHIPKernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Cos,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n CosFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n CosGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n CosGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"453 },454 {455 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/counter_ops.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(CreateCounter, CreateCounterOp<int64_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(ResetCounter, ResetCounterOp<int64_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(CountDown, CountDownOp<int64_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n CheckCounterDone,\n CheckCounterDoneOp<int64_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(CountUp, CountUpOp<int64_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(RetrieveCount, RetrieveCountOp<int64_t, CUDAContext>);\n} // namespace caffe2\n\n\n###",456 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/counter_ops.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(CreateCounter, CreateCounterOp<int64_t, HIPContext>);\nREGISTER_HIP_OPERATOR(ResetCounter, ResetCounterOp<int64_t, HIPContext>);\nREGISTER_HIP_OPERATOR(CountDown, CountDownOp<int64_t, HIPContext>);\nREGISTER_HIP_OPERATOR(\n CheckCounterDone,\n CheckCounterDoneOp<int64_t, HIPContext>);\nREGISTER_HIP_OPERATOR(CountUp, CountUpOp<int64_t, HIPContext>);\nREGISTER_HIP_OPERATOR(RetrieveCount, RetrieveCountOp<int64_t, HIPContext>);\n} // namespace caffe2\n###"457 },458 {459 "cuda": "\n#include \"caffe2/operators/cube_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nCubeGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * __ldg(X + i) * __ldg(X + i) * T(3);\n#else\n dX[i] = dY[i] * X[i] * X[i] * T(3);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CubeGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& dY_dims,\n const std::vector<int>& /* X_dims */,\n const T* dY,\n const T* X,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n dY_dims.cbegin(), dY_dims.cend(), 1, std::multiplies<int>());\n CubeGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Cube,\n UnaryElementwiseOp<NumericTypes, CUDAContext, CubeFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n CubeGradient,\n BinaryElementwiseOp<\n NumericTypes,\n CUDAContext,\n CubeGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",460 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/cube_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nCubeGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * __ldg(X + i) * __ldg(X + i) * T(3);\n#else\n dX[i] = dY[i] * X[i] * X[i] * T(3);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool CubeGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& dY_dims,\n const std::vector<int>& /* X_dims */,\n const T* dY,\n const T* X,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n dY_dims.cbegin(), dY_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( CubeGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Cube,\n UnaryElementwiseOp<NumericTypes, HIPContext, CubeFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n CubeGradient,\n BinaryElementwiseOp<\n NumericTypes,\n HIPContext,\n CubeGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"461 },462 {463 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/data_couple.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(DataCouple, DataCoupleOp<CUDAContext>);\n}\n\n\n###",464 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/data_couple.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(DataCouple, DataCoupleOp<HIPContext>);\n}\n###"465 },466 {467 "cuda": "\n#include \"caffe2/operators/do_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Do, DoOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",468 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/do_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Do, DoOp<HIPContext>);\n\n} // namespace caffe2\n###"469 },470 {471 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/dropout_op.h\"\nnamespace caffe2 {\nnamespace {\n__global__ void DropoutKernel(\n const int N, const float ratio, const float* Xdata, float* Ydata, bool* maskdata) {\n const float scale = 1. / (1. - ratio);\n CUDA_1D_KERNEL_LOOP(i, N) {\n maskdata[i] = (Ydata[i] > ratio);\n Ydata[i] = Xdata[i] * scale * maskdata[i];\n }\n}\n} \ntemplate <>\nbool DropoutOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n if (is_test_) {\n if (Y != &X) {\n context_.CopySameDevice<float>(\n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n }\n return true;\n } else {\n \n \n \n float* Ydata = Y->template mutable_data<float>();\n auto* mask = Output(1, X.sizes(), at::dtype<bool>());\n CAFFE_ENFORCE(X.data<float>() != Ydata, \"In-place GPU dropout is broken\");\n CURAND_ENFORCE(\n curandGenerateUniform(context_.curand_generator(), Ydata, X.numel()));\n DropoutKernel<<<\n CAFFE_GET_BLOCKS(X.numel()), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n X.numel(), ratio_, X.data<float>(), Ydata, mask->template mutable_data<bool>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return true;\n }\n}\nnamespace {\n__global__ void DropoutGradientKernel(\n const int N, const float* dYdata, const bool* maskdata, const float scale, float* dXdata) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n dXdata[i] = dYdata[i] * maskdata[i] * scale;\n }\n}\n} \ntemplate <>\nbool DropoutGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& dY = Input(0);\n auto* dX = Output(0, dY.sizes(), at::dtype<float>());\n if (is_test_) {\n if (dX != &dY) {\n context_.CopySameDevice<float>(\n dY.numel(), dY.data<float>(), dX->template mutable_data<float>());\n }\n return true;\n } else {\n auto& mask = Input(1);\n CAFFE_ENFORCE_EQ(dY.numel(), mask.numel());\n const float scale = 1. / (1. - ratio_);\n DropoutGradientKernel<<<\n CAFFE_GET_BLOCKS(dY.numel()), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n dY.numel(), dY.data<float>(), mask.data<bool>(), scale, dX->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return true;\n }\n}\nREGISTER_CUDA_OPERATOR(Dropout, DropoutOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(DropoutGrad, DropoutGradientOp<float, CUDAContext>);\n} \n\n###",472 "hip": " \n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/dropout_op.h\"\nnamespace caffe2 {\nnamespace {\n__global__ void DropoutKernel(\n const int N, const float ratio, const float* Xdata, float* Ydata, bool* maskdata) {\n const float scale = 1. / (1. - ratio);\n HIP_1D_KERNEL_LOOP(i, N) {\n maskdata[i] = (Ydata[i] > ratio);\n Ydata[i] = Xdata[i] * scale * maskdata[i];\n }\n}\n} \ntemplate <>\nbool DropoutOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n if (is_test_) {\n if (Y != &X) {\n context_.CopySameDevice<float>(\n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n }\n return true;\n } else {\n \n \n \n float* Ydata = Y->template mutable_data<float>();\n auto* mask = Output(1, X.sizes(), at::dtype<bool>());\n CAFFE_ENFORCE(X.data<float>() != Ydata, \"In-place GPU dropout is broken\");\n HIPRAND_ENFORCE(\n hiprandGenerateUniform(context_.hiprand_generator(), Ydata, X.numel()));\n hipLaunchKernelGGL(( DropoutKernel), dim3(CAFFE_GET_BLOCKS(X.numel())), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), X.numel(), ratio_, X.data<float>(), Ydata, mask->template mutable_data<bool>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n return true;\n }\n}\nnamespace {\n__global__ void DropoutGradientKernel(\n const int N, const float* dYdata, const bool* maskdata, const float scale, float* dXdata) {\n HIP_1D_KERNEL_LOOP(i, N) {\n dXdata[i] = dYdata[i] * maskdata[i] * scale;\n }\n}\n} \ntemplate <>\nbool DropoutGradientOp<float, HIPContext>::RunOnDevice() {\n auto& dY = Input(0);\n auto* dX = Output(0, dY.sizes(), at::dtype<float>());\n if (is_test_) {\n if (dX != &dY) {\n context_.CopySameDevice<float>(\n dY.numel(), dY.data<float>(), dX->template mutable_data<float>());\n }\n return true;\n } else {\n auto& mask = Input(1);\n CAFFE_ENFORCE_EQ(dY.numel(), mask.numel());\n const float scale = 1. / (1. - ratio_);\n hipLaunchKernelGGL(( DropoutGradientKernel), dim3(CAFFE_GET_BLOCKS(dY.numel())), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), dY.numel(), dY.data<float>(), mask.data<bool>(), scale, dX->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n return true;\n }\n}\nREGISTER_HIP_OPERATOR(Dropout, DropoutOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(DropoutGrad, DropoutGradientOp<float, HIPContext>);\n} ###"473 },474 {475 "cuda": "\n#include \"caffe2/operators/elementwise_add_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Add,\n BinaryElementwiseOp<NumericTypes, CUDAContext, AddFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n AddGradient,\n BinaryElementwiseGradientOp<\n NumericTypes,\n CUDAContext,\n AddFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",476 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/elementwise_add_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Add,\n BinaryElementwiseOp<NumericTypes, HIPContext, AddFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n AddGradient,\n BinaryElementwiseGradientOp<\n NumericTypes,\n HIPContext,\n AddFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"477 },478 {479 "cuda": "\n#include \"caffe2/operators/elementwise_op_test.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/flags.h\"\n\nC10_DECLARE_string(caffe_test_root);\n\ntemplate <>\nvoid CopyVector<caffe2::CUDAContext>(const int N, const bool* x, bool* y) {\n CUDA_CHECK(cudaMemcpy(y, x, N * sizeof(bool), cudaMemcpyHostToDevice));\n}\n\ntemplate <>\ncaffe2::OperatorDef CreateOperatorDef<caffe2::CUDAContext>() {\n caffe2::OperatorDef def;\n def.mutable_device_option()->set_device_type(caffe2::PROTO_CUDA);\n return def;\n}\n\nTEST(ElementwiseGPUTest, And) {\n if (!caffe2::HasCudaGPU())\n return;\n elementwiseAnd<caffe2::CUDAContext>();\n}\n\nTEST(ElementwiseGPUTest, Or) {\n if (!caffe2::HasCudaGPU())\n return;\n elementwiseOr<caffe2::CUDAContext>();\n}\n\nTEST(ElementwiseGPUTest, Xor) {\n if (!caffe2::HasCudaGPU())\n return;\n elementwiseXor<caffe2::CUDAContext>();\n}\n\nTEST(ElementwiseGPUTest, Not) {\n if (!caffe2::HasCudaGPU())\n return;\n elementwiseNot<caffe2::CUDAContext>();\n}\n\n\n###",480 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/elementwise_op_test.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/flags.h\"\n\nC10_DECLARE_string(caffe_test_root);\n\ntemplate <>\nvoid CopyVector<caffe2::HIPContext>(const int N, const bool* x, bool* y) {\n HIP_CHECK(hipMemcpy(y, x, N * sizeof(bool), hipMemcpyHostToDevice));\n}\n\ntemplate <>\ncaffe2::OperatorDef CreateOperatorDef<caffe2::HIPContext>() {\n caffe2::OperatorDef def;\n def.mutable_device_option()->set_device_type(caffe2::PROTO_HIP);\n return def;\n}\n\nTEST(ElementwiseGPUTest, And) {\n if (!caffe2::HasHipGPU())\n return;\n elementwiseAnd<caffe2::HIPContext>();\n}\n\nTEST(ElementwiseGPUTest, Or) {\n if (!caffe2::HasHipGPU())\n return;\n elementwiseOr<caffe2::HIPContext>();\n}\n\nTEST(ElementwiseGPUTest, Xor) {\n if (!caffe2::HasHipGPU())\n return;\n elementwiseXor<caffe2::HIPContext>();\n}\n\nTEST(ElementwiseGPUTest, Not) {\n if (!caffe2::HasHipGPU())\n return;\n elementwiseNot<caffe2::HIPContext>();\n}\n###"481 },482 {483 "cuda": "\n#include \"caffe2/operators/elementwise_sub_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Sub,\n BinaryElementwiseOp<NumericTypes, CUDAContext, SubFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n SubGradient,\n BinaryElementwiseGradientOp<\n NumericTypes,\n CUDAContext,\n SubFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",484 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/elementwise_sub_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Sub,\n BinaryElementwiseOp<NumericTypes, HIPContext, SubFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n SubGradient,\n BinaryElementwiseGradientOp<\n NumericTypes,\n HIPContext,\n SubFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"485 },486 {487 "cuda": "\n#include \"caffe2/operators/elu_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void EluCUDAKernel(const int N, const T alpha, const T* X, T* Y);\n\ntemplate <>\n__global__ void\nEluCUDAKernel<float>(const int N, const float alpha, const float* X, float* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n Y[i] =\n __ldg(X + i) < 0 ? alpha * (expf(__ldg(X + i)) - 1.0f) : __ldg(X + i);\n#else\n Y[i] = X[i] < 0 ? alpha * (expf(X[i]) - 1.0f) : X[i];\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void EluGradientCUDAKernel(\n const int N,\n const T alpha,\n const T* dY,\n const T* Y,\n T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(Y + i) < 0 ? __ldg(dY + i) * (__ldg(Y + i) + alpha)\n : __ldg(dY + i);\n#else\n dX[i] = Y[i] < 0 ? dY[i] * (Y[i] + alpha) : dY[i];\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool EluFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n EluCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, alpha, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool EluGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n EluGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, alpha, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Elu,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n EluFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n EluGradient,\n BinaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n EluGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",488 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/elu_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void EluHIPKernel(const int N, const T alpha, const T* X, T* Y);\n\ntemplate <>\n__global__ void\nEluHIPKernel<float>(const int N, const float alpha, const float* X, float* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n Y[i] =\n __ldg(X + i) < 0 ? alpha * (expf(__ldg(X + i)) - 1.0f) : __ldg(X + i);\n#else\n Y[i] = X[i] < 0 ? alpha * (expf(X[i]) - 1.0f) : X[i];\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void EluGradientHIPKernel(\n const int N,\n const T alpha,\n const T* dY,\n const T* Y,\n T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(Y + i) < 0 ? __ldg(dY + i) * (__ldg(Y + i) + alpha)\n : __ldg(dY + i);\n#else\n dX[i] = Y[i] < 0 ? dY[i] * (Y[i] + alpha) : dY[i];\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool EluFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( EluHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), N, alpha, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool EluGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( EluGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, alpha, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Elu,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n EluFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n EluGradient,\n BinaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n EluGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"489 },490 {491 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardtanh_backward_kernel(\n TensorIterator& iter,\n const Scalar& min,\n const Scalar& max) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half, at::ScalarType::BFloat16,\n iter.dtype(), \"hardtanh_backward_cuda\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto min_val = min.to<opmath_t>();\n auto max_val = max.to<opmath_t>();\n gpu_kernel(\n iter,\n [min_val, max_val] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n return (bop <= min_val) || (bop >= max_val) ? opmath_t(0) : aop;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(hardtanh_backward_stub, &hardtanh_backward_kernel);\n\n} // namespace at::native\n\n\n###",492 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid hardtanh_backward_kernel(\n TensorIterator& iter,\n const Scalar& min,\n const Scalar& max) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half, at::ScalarType::BFloat16,\n iter.dtype(), \"hardtanh_backward_hip\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto min_val = min.to<opmath_t>();\n auto max_val = max.to<opmath_t>();\n gpu_kernel(\n iter,\n [min_val, max_val] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n return (bop <= min_val) || (bop >= max_val) ? opmath_t(0) : aop;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(hardtanh_backward_stub, &hardtanh_backward_kernel);\n\n} // namespace at::native\n###"493 },494 {495 "cuda": "\n#include \"caffe2/operators/enforce_finite_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\ntemplate <>\ntemplate <typename T>\nbool EnforceFiniteOp<CUDAContext>::DoRunWithType() {\n buffer_.CopyFrom(Input(0)); // sync copy\n EnforceOnCPU<T>(buffer_);\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(EnforceFinite, EnforceFiniteOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",496 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/enforce_finite_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\ntemplate <>\ntemplate <typename T>\nbool EnforceFiniteOp<HIPContext>::DoRunWithType() {\n buffer_.CopyFrom(Input(0)); // sync copy\n EnforceOnCPU<T>(buffer_);\n return true;\n}\n\nREGISTER_HIP_OPERATOR(EnforceFinite, EnforceFiniteOp<HIPContext>);\n} // namespace caffe2\n###"497 },498 {499 "cuda": "\n#include \"caffe2/operators/ensure_cpu_output_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n// From CUDA Context, takes either CUDA or CPU tensor as input, and produce\n// TensorCPU\nREGISTER_CUDA_OPERATOR(EnsureCPUOutput, EnsureCPUOutputOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",500 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/ensure_cpu_output_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n// From HIP Context, takes either HIP or CPU tensor as input, and produce\n// TensorCPU\nREGISTER_HIP_OPERATOR(EnsureCPUOutput, EnsureCPUOutputOp<HIPContext>);\n} // namespace caffe2\n###"501 },502 {503 "cuda": "\n#include \"caffe2/operators/erf_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void ErfGradientCUDAKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = 2.0f / sqrtf(PI) * expf(-powf(__ldg(X+i), 2.0f)) * __ldg(dY + i);\n#else\n dX[i] = 2.0f / sqrtf(PI) * expf(-powf(X[i], 2.0f)) * dY[i];\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ErfGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n ErfGradientCUDAKernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Erf,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n ErfFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n ErfGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n ErfGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",504 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/erf_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void ErfGradientHIPKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = 2.0f / sqrtf(PI) * expf(-powf(__ldg(X+i), 2.0f)) * __ldg(dY + i);\n#else\n dX[i] = 2.0f / sqrtf(PI) * expf(-powf(X[i], 2.0f)) * dY[i];\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ErfGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( ErfGradientHIPKernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Erf,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n ErfFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n ErfGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n ErfGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"505 },506 {507 "cuda": "\n#include \"caffe2/operators/expand_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Expand,\n ExpandOp<\n TensorTypes<std::int32_t, std::int64_t, float, double>,\n CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n ExpandGradient,\n ExpandGradientOp<\n TensorTypes<std::int32_t, std::int64_t, float, double>,\n CUDAContext>);\n} // namespace caffe2\n\n\n###",508 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/expand_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Expand,\n ExpandOp<\n TensorTypes<std::int32_t, std::int64_t, float, double>,\n HIPContext>);\nREGISTER_HIP_OPERATOR(\n ExpandGradient,\n ExpandGradientOp<\n TensorTypes<std::int32_t, std::int64_t, float, double>,\n HIPContext>);\n} // namespace caffe2\n###"509 },510 {511 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/expand_squeeze_dims_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(Squeeze, SqueezeOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(ExpandDims, ExpandDimsOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",512 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/expand_squeeze_dims_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(Squeeze, SqueezeOp<HIPContext>);\nREGISTER_HIP_OPERATOR(ExpandDims, ExpandDimsOp<HIPContext>);\n} // namespace caffe2\n###"513 },514 {515 "cuda": "\n#include \"caffe2/operators/exp_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Exp,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n ExpFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",516 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/exp_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Exp,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n ExpFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"517 },518 {519 "cuda": "\n#include <cmath>\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/filler_op.h\"\n#include \"caffe2/operators/operator_fallback_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void FillRangeKernel(const int n, float* data) {\n CUDA_1D_KERNEL_LOOP(index, n) {\n data[index] = index;\n }\n}\n\ntemplate <typename T>\n__global__ void FillDiagonalKernel(\n const int num_diagonal_elements,\n const int64_t step_size,\n const T value,\n T* data) {\n CUDA_1D_KERNEL_LOOP(index, num_diagonal_elements) {\n data[index * step_size] = value;\n }\n}\n}\n\ntemplate <>\nbool RangeFillOp<float, CUDAContext>::Fill(Tensor* output) {\n int N = output->numel();\n FillRangeKernel<<<\n CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(N, output->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool DiagonalFillOp<CUDAContext>::FillWithType(Tensor* output) {\n VerifyOutputShape(output);\n auto* data = output->template mutable_data<T>();\n int size = output->numel();\n // first fill everything with 0\n math::Set<T, CUDAContext>(size, T(0), data, &context_);\n\n T value = OperatorBase::GetSingleArgument<T>(\"value\", 0);\n int64_t step_size = GetStepSize(output);\n int num_diagonal_elements = ceil((float)size / step_size);\n\n FillDiagonalKernel<<<\n CAFFE_GET_BLOCKS(num_diagonal_elements),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(num_diagonal_elements, step_size, value, data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(UniformFill, UniformFillOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(UniformIntFill, UniformFillOp<int, CUDAContext>);\nREGISTER_CUDA_OPERATOR(ConstantFill, ConstantFillOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(DiagonalFill, DiagonalFillOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(GaussianFill, GaussianFillOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(XavierFill, XavierFillOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(MSRAFill, MSRAFillOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(RangeFill, RangeFillOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(LengthsRangeFill, GPUFallbackOp);\n\n} // namespace caffe2\n\n\n###",520 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <cmath>\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/filler_op.h\"\n#include \"caffe2/operators/hip/operator_fallback_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void FillRangeKernel(const int n, float* data) {\n HIP_1D_KERNEL_LOOP(index, n) {\n data[index] = index;\n }\n}\n\ntemplate <typename T>\n__global__ void FillDiagonalKernel(\n const int num_diagonal_elements,\n const int64_t step_size,\n const T value,\n T* data) {\n HIP_1D_KERNEL_LOOP(index, num_diagonal_elements) {\n data[index * step_size] = value;\n }\n}\n}\n\ntemplate <>\nbool RangeFillOp<float, HIPContext>::Fill(Tensor* output) {\n int N = output->numel();\n hipLaunchKernelGGL(( FillRangeKernel), \n dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), N, output->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool DiagonalFillOp<HIPContext>::FillWithType(Tensor* output) {\n VerifyOutputShape(output);\n auto* data = output->template mutable_data<T>();\n int size = output->numel();\n // first fill everything with 0\n math::Set<T, HIPContext>(size, T(0), data, &context_);\n\n T value = OperatorBase::GetSingleArgument<T>(\"value\", 0);\n int64_t step_size = GetStepSize(output);\n int num_diagonal_elements = ceil((float)size / step_size);\n\n hipLaunchKernelGGL(( FillDiagonalKernel), \n dim3(CAFFE_GET_BLOCKS(num_diagonal_elements)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), num_diagonal_elements, step_size, value, data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(UniformFill, UniformFillOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(UniformIntFill, UniformFillOp<int, HIPContext>);\nREGISTER_HIP_OPERATOR(ConstantFill, ConstantFillOp<HIPContext>);\nREGISTER_HIP_OPERATOR(DiagonalFill, DiagonalFillOp<HIPContext>);\nREGISTER_HIP_OPERATOR(GaussianFill, GaussianFillOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(XavierFill, XavierFillOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(MSRAFill, MSRAFillOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(RangeFill, RangeFillOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(LengthsRangeFill, GPUFallbackOp);\n\n} // namespace caffe2\n###"521 },522 {523 "cuda": "\n#include <cub/block/block_reduce.cuh>\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/find_op.h\"\n#include \"caffe2/utils/cub_namespace.cuh\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\n__global__ void FindKernel(\n int num_needles,\n int idx_size,\n const T* idx,\n const T* needles,\n int* out,\n int missing_value) {\n int needle_idx = blockIdx.x; // One cuda block per needle\n T q = needles[needle_idx];\n int res = (-1);\n for (int j = threadIdx.x; j < idx_size; j += CAFFE_CUDA_NUM_THREADS) {\n if (idx[j] == q) {\n res = max(res, j);\n }\n }\n typedef cub::BlockReduce<int, CAFFE_CUDA_NUM_THREADS> BlockReduce;\n __shared__ typename BlockReduce::TempStorage temp_storage;\n int min_res = BlockReduce(temp_storage).Reduce(res, cub::Max());\n if (threadIdx.x == 0) {\n out[needle_idx] = min_res == (-1) ? missing_value : min_res;\n }\n}\n\ntemplate <>\ntemplate <typename T>\nbool FindOp<CUDAContext>::DoRunWithType() {\n auto& idx = Input(0);\n auto& needles = Input(1);\n\n auto* res_indices = Output(0, needles.sizes(), at::dtype<int>());\n\n const T* idx_data = idx.data<T>();\n const T* needles_data = needles.data<T>();\n int* res_data = res_indices->template mutable_data<int>();\n\n FindKernel<\n T><<<needles.numel(), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n needles.numel(),\n idx.numel(),\n idx_data,\n needles_data,\n res_data,\n missing_value_);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Find, FindOp<CUDAContext>)\n\n} // namespace caffe2\n\n\n###",524 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <hipcub/hipcub.hpp>\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/find_op.h\"\n#include \"caffe2/utils/cub_namespace.cuh\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\n__global__ void FindKernel(\n int num_needles,\n int idx_size,\n const T* idx,\n const T* needles,\n int* out,\n int missing_value) {\n int needle_idx = blockIdx.x; // One cuda block per needle\n T q = needles[needle_idx];\n int res = (-1);\n for (int j = threadIdx.x; j < idx_size; j += CAFFE_HIP_NUM_THREADS) {\n if (idx[j] == q) {\n res = max(res, j);\n }\n }\n typedef hipcub::BlockReduce<int, CAFFE_HIP_NUM_THREADS> BlockReduce;\n __shared__ typename BlockReduce::TempStorage temp_storage;\n int min_res = BlockReduce(temp_storage).Reduce(res, hipcub::Max());\n if (threadIdx.x == 0) {\n out[needle_idx] = min_res == (-1) ? missing_value : min_res;\n }\n}\n\ntemplate <>\ntemplate <typename T>\nbool FindOp<HIPContext>::DoRunWithType() {\n auto& idx = Input(0);\n auto& needles = Input(1);\n\n auto* res_indices = Output(0, needles.sizes(), at::dtype<int>());\n\n const T* idx_data = idx.data<T>();\n const T* needles_data = needles.data<T>();\n int* res_data = res_indices->template mutable_data<int>();\n\n hipLaunchKernelGGL(( FindKernel<\n T>), dim3(needles.numel()), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), \n needles.numel(),\n idx.numel(),\n idx_data,\n needles_data,\n res_data,\n missing_value_);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Find, FindOp<HIPContext>)\n\n} // namespace caffe2\n###"525 },526 {527 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/floor_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\n__global__ void FloorKernel(const int N, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = std::floor(X[i]);\n }\n}\n\ntemplate <>\nbool FloorOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n FloorKernel<<<\n CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Floor, FloorOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",528 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/floor_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\n__global__ void FloorKernel(const int N, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = ::floor(X[i]);\n }\n}\n\ntemplate <>\nbool FloorOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( FloorKernel), \n dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Floor, FloorOp<float, HIPContext>);\n\n} // namespace caffe2\n###"529 },530 {531 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/free_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(Free, FreeOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",532 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/free_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(Free, FreeOp<HIPContext>);\n} // namespace caffe2\n###"533 },534 {535 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid leaky_relu_kernel(TensorIteratorBase& iter, const Scalar& negval_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"leaky_relu_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negval = negval_.to<opmath_t>();\n gpu_kernel(iter, [negval] GPU_LAMBDA(scalar_t a) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n return aop > opmath_t(0) ? aop : aop * negval;\n });\n });\n}\n\nvoid leaky_relu_backward_kernel(\n TensorIteratorBase& iter,\n const Scalar& negval_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"leaky_relu_backward_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negval = negval_.to<opmath_t>();\n gpu_kernel(\n iter, [negval] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n return aop > opmath_t(0) ? bop : bop * negval;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(leaky_relu_stub, &leaky_relu_kernel);\nREGISTER_DISPATCH(leaky_relu_backward_stub, &leaky_relu_backward_kernel);\n\n} // namespace at::native\n\n\n###",536 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid leaky_relu_kernel(TensorIteratorBase& iter, const Scalar& negval_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"leaky_relu_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negval = negval_.to<opmath_t>();\n gpu_kernel(iter, [negval] GPU_LAMBDA(scalar_t a) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n return aop > opmath_t(0) ? aop : aop * negval;\n });\n });\n}\n\nvoid leaky_relu_backward_kernel(\n TensorIteratorBase& iter,\n const Scalar& negval_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"leaky_relu_backward_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto negval = negval_.to<opmath_t>();\n gpu_kernel(\n iter, [negval] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n return aop > opmath_t(0) ? bop : bop * negval;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(leaky_relu_stub, &leaky_relu_kernel);\nREGISTER_DISPATCH(leaky_relu_backward_stub, &leaky_relu_backward_kernel);\n\n} // namespace at::native\n###"537 },538 {539 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/gather_op.h\"\n#include \"caffe2/operators/gather_op.cuh\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool GatherOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<int32_t, int64_t>>::call(\n this, OperatorBase::Input<Tensor>(INDICES, CUDA));\n}\n\ntemplate <>\ntemplate <typename Index>\nbool GatherOp<CUDAContext>::DoRunWithType() {\n // Use shared implementation with BatchGather\n return gather_helper::gather_impl_cuda<Index>(\n this, DATA, INDICES, 0, axis_, wrap_indices_, match_outer_);\n}\n\nREGISTER_CUDA_OPERATOR(Gather, GatherOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",540 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/gather_op.h\"\n#include \"caffe2/operators/hip/gather_op.cuh\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool GatherOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<int32_t, int64_t>>::call(\n this, OperatorBase::Input<Tensor>(INDICES, HIP));\n}\n\ntemplate <>\ntemplate <typename Index>\nbool GatherOp<HIPContext>::DoRunWithType() {\n // Use shared implementation with BatchGather\n return gather_helper::gather_impl_hip<Index>(\n this, DATA, INDICES, 0, axis_, wrap_indices_, match_outer_);\n}\n\nREGISTER_HIP_OPERATOR(Gather, GatherOp<HIPContext>);\n} // namespace caffe2\n###"541 },542 {543 "cuda": "\n#ifndef CAFFE2_OPERATORS_UTILS_NMS_GPU_H_\n#define CAFFE2_OPERATORS_UTILS_NMS_GPU_H_\n\n#include <vector>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\nnamespace utils {\n\n// Computes Non-Maximum Suppression on the GPU\n// Reject a bounding box if its region has an intersection-overunion (IoU)\n// overlap with a higher scoring selected bounding box larger than a\n// threshold.\n//\n// d_desc_sorted_boxes : pixel coordinates of proposed bounding boxes\n// size: (N,4), format: [x1; y1; x2; y2]\n// the boxes are sorted by scores in descending order\n// N : number of boxes\n// d_keep_sorted_list : row indices of the selected proposals, sorted by score\n// h_nkeep : number of selected proposals\n// dev_delete_mask, host_delete_mask : Tensors that will be used as temp storage\n// by NMS\n// Those tensors will be resized to the necessary size\n// context : current CUDA context\nTORCH_API void nms_gpu_upright(\n const float* d_desc_sorted_boxes,\n const int N,\n const float thresh,\n const bool legacy_plus_one,\n int* d_keep_sorted_list,\n int* h_nkeep,\n TensorCUDA& dev_delete_mask,\n TensorCPU& host_delete_mask,\n CUDAContext* context);\n\nstruct RotatedBox {\n float x_ctr, y_ctr, w, h, a;\n};\n\n// Same as nms_gpu_upright, but for rotated boxes with angle info.\n// d_desc_sorted_boxes : pixel coordinates of proposed bounding boxes\n// size: (N,5), format: [x_ct; y_ctr; width; height; angle]\n// the boxes are sorted by scores in descending order\nTORCH_API void nms_gpu_rotated(\n const float* d_desc_sorted_boxes,\n const int N,\n const float thresh,\n int* d_keep_sorted_list,\n int* h_nkeep,\n TensorCUDA& dev_delete_mask,\n TensorCPU& host_delete_mask,\n CUDAContext* context);\n\nTORCH_API void nms_gpu(\n const float* d_desc_sorted_boxes,\n const int N,\n const float thresh,\n const bool legacy_plus_one,\n int* d_keep_sorted_list,\n int* h_nkeep,\n TensorCUDA& dev_delete_mask,\n TensorCPU& host_delete_mask,\n CUDAContext* context,\n const int box_dim);\n\n} // namespace utils\n} // namespace caffe2\n\n#endif // CAFFE2_OPERATORS_UTILS_NMS_GPU_H_\n\n\n###",544 "hip": " // !!! This is a file automatically generated by hipify!!!\n#ifndef CAFFE2_OPERATORS_UTILS_NMS_GPU_H_\n#define CAFFE2_OPERATORS_UTILS_NMS_GPU_H_\n\n#include <vector>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\nnamespace utils {\n\n// Computes Non-Maximum Suppression on the GPU\n// Reject a bounding box if its region has an intersection-overunion (IoU)\n// overlap with a higher scoring selected bounding box larger than a\n// threshold.\n//\n// d_desc_sorted_boxes : pixel coordinates of proposed bounding boxes\n// size: (N,4), format: [x1; y1; x2; y2]\n// the boxes are sorted by scores in descending order\n// N : number of boxes\n// d_keep_sorted_list : row indices of the selected proposals, sorted by score\n// h_nkeep : number of selected proposals\n// dev_delete_mask, host_delete_mask : Tensors that will be used as temp storage\n// by NMS\n// Those tensors will be resized to the necessary size\n// context : current HIP context\nTORCH_API void nms_gpu_upright(\n const float* d_desc_sorted_boxes,\n const int N,\n const float thresh,\n const bool legacy_plus_one,\n int* d_keep_sorted_list,\n int* h_nkeep,\n TensorHIP& dev_delete_mask,\n TensorCPU& host_delete_mask,\n HIPContext* context);\n\nstruct RotatedBox {\n float x_ctr, y_ctr, w, h, a;\n};\n\n// Same as nms_gpu_upright, but for rotated boxes with angle info.\n// d_desc_sorted_boxes : pixel coordinates of proposed bounding boxes\n// size: (N,5), format: [x_ct; y_ctr; width; height; angle]\n// the boxes are sorted by scores in descending order\nTORCH_API void nms_gpu_rotated(\n const float* d_desc_sorted_boxes,\n const int N,\n const float thresh,\n int* d_keep_sorted_list,\n int* h_nkeep,\n TensorHIP& dev_delete_mask,\n TensorCPU& host_delete_mask,\n HIPContext* context);\n\nTORCH_API void nms_gpu(\n const float* d_desc_sorted_boxes,\n const int N,\n const float thresh,\n const bool legacy_plus_one,\n int* d_keep_sorted_list,\n int* h_nkeep,\n TensorHIP& dev_delete_mask,\n TensorCPU& host_delete_mask,\n HIPContext* context,\n const int box_dim);\n\n} // namespace utils\n} // namespace caffe2\n\n#endif // CAFFE2_OPERATORS_UTILS_NMS_GPU_H_\n###"545 },546 {547 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/given_tensor_byte_string_to_uint8_fill_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n GivenTensorByteStringToUInt8Fill,\n GivenTensorByteStringToUInt8FillOp<CUDAContext>);\n}\n\n\n###",548 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/given_tensor_byte_string_to_uint8_fill_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n GivenTensorByteStringToUInt8Fill,\n GivenTensorByteStringToUInt8FillOp<HIPContext>);\n}\n###"549 },550 {551 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/given_tensor_fill_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(GivenTensorFill, GivenTensorFillOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n GivenTensorDoubleFill,\n GivenTensorFillOp<double, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n GivenTensorInt16Fill,\n GivenTensorFillOp<int16_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(GivenTensorIntFill, GivenTensorFillOp<int, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n GivenTensorInt64Fill,\n GivenTensorFillOp<int64_t, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n GivenTensorBoolFill,\n GivenTensorFillOp<bool, CUDAContext>);\n}\n\n\n###",552 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/given_tensor_fill_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(GivenTensorFill, GivenTensorFillOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n GivenTensorDoubleFill,\n GivenTensorFillOp<double, HIPContext>);\nREGISTER_HIP_OPERATOR(\n GivenTensorInt16Fill,\n GivenTensorFillOp<int16_t, HIPContext>);\nREGISTER_HIP_OPERATOR(GivenTensorIntFill, GivenTensorFillOp<int, HIPContext>);\nREGISTER_HIP_OPERATOR(\n GivenTensorInt64Fill,\n GivenTensorFillOp<int64_t, HIPContext>);\nREGISTER_HIP_OPERATOR(\n GivenTensorBoolFill,\n GivenTensorFillOp<bool, HIPContext>);\n}\n###"553 },554 {555 "cuda": "\n#include \"caffe2/operators/glu_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void glu_kernel(\n const int M,\n const int split_dim_size,\n const int N,\n const float* Xdata,\n float* Ydata) {\n const int xOffset = 2 * split_dim_size * N;\n const int yOffset = split_dim_size * N;\n CUDA_1D_KERNEL_LOOP(index, M * split_dim_size * N) {\n const int i = index / split_dim_size / N;\n const int j = index / N % split_dim_size;\n const int k = index % N;\n const float x1 = Xdata[i * xOffset + j * N + k];\n const float x2 = Xdata[i * xOffset + (j + split_dim_size) * N + k];\n Ydata[i * yOffset + j * N + k] = x1 * (1. / (1. + exp(-x2)));\n }\n}\n} // namespace\n\ntemplate <>\nvoid GluOp<float, CUDAContext>::ComputeGlu(\n const int M,\n const int split_dim_size,\n const int N,\n const float* x_data,\n float* y_data) {\n glu_kernel<<<\n CAFFE_GET_BLOCKS(M * N * split_dim_size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(M, split_dim_size, N, x_data, y_data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_CUDA_OPERATOR(Glu, GluOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",556 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/glu_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void glu_kernel(\n const int M,\n const int split_dim_size,\n const int N,\n const float* Xdata,\n float* Ydata) {\n const int xOffset = 2 * split_dim_size * N;\n const int yOffset = split_dim_size * N;\n HIP_1D_KERNEL_LOOP(index, M * split_dim_size * N) {\n const int i = index / split_dim_size / N;\n const int j = index / N % split_dim_size;\n const int k = index % N;\n const float x1 = Xdata[i * xOffset + j * N + k];\n const float x2 = Xdata[i * xOffset + (j + split_dim_size) * N + k];\n Ydata[i * yOffset + j * N + k] = x1 * (1. / (1. + exp(-x2)));\n }\n}\n} // namespace\n\ntemplate <>\nvoid GluOp<float, HIPContext>::ComputeGlu(\n const int M,\n const int split_dim_size,\n const int N,\n const float* x_data,\n float* y_data) {\n hipLaunchKernelGGL(( glu_kernel), \n dim3(CAFFE_GET_BLOCKS(M * N * split_dim_size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), M, split_dim_size, N, x_data, y_data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_HIP_OPERATOR(Glu, GluOp<float, HIPContext>);\n} // namespace caffe2\n###"557 },558 {559 "cuda": "\n#include \"caffe2/operators/half_float_ops.h\"\n#include \"caffe2/core/context_gpu.h\"\n#ifdef CAFFE_HAS_CUDA_FP16\nnamespace caffe2 {\nnamespace {\n__global__ void FloatToHalfKernel(const int N, const float* X, half* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = __float2half(X[i]);\n }\n}\n__global__ void HalfToFloatKernel(const int N, const half* X, float* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = __half2float(X[i]);\n }\n}\n}\ntemplate <>\nbool FloatToHalfOp<CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n auto* Y = Output(0, X.sizes(), at::dtype<at::Half>());\n FloatToHalfKernel<<<\n CAFFE_GET_BLOCKS(X.numel()), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n X.numel(), X.data<float>(), reinterpret_cast<half*>(Y->template mutable_data<at::Half>()));\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return true;\n}\ntemplate <>\nbool HalfToFloatOp<CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n HalfToFloatKernel<<<\n CAFFE_GET_BLOCKS(X.numel()), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n X.numel(), reinterpret_cast<const half*>(X.data<at::Half>()), Y->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n return true;\n}\ntemplate <>\nbool Float16UniformFillOp<CUDAContext>::RunOnDevice() {\n auto* output = Output(0, shape_, at::dtype<at::Half>());\n at::Half* out = output->template mutable_data<at::Half>();\n auto leading_dim_sz = output->size(0);\n CAFFE_ENFORCE_GT(leading_dim_sz, 0, \"The input shape should have the first dimension greater than 0\");\n int rowsz = output->numel() / output->size(0);\n ReinitializeTensor(\n &temp_data_buffer_, {rowsz}, at::dtype<float>().device(CUDA));\n float* temp_data = temp_data_buffer_.template mutable_data<float>();\n for (uint64_t i = 0; i < leading_dim_sz; i++) {\n math::RandUniform<float, CUDAContext>(\n rowsz, min_, max_, temp_data, &context_);\n FloatToHalfKernel<<<\n CAFFE_GET_BLOCKS(rowsz), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n rowsz, temp_data, reinterpret_cast<half*>(out + i * rowsz));\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n }\n return true;\n}\nREGISTER_CUDA_OPERATOR(FloatToHalf, FloatToHalfOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(HalfToFloat, HalfToFloatOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Float16UniformFill, Float16UniformFillOp<CUDAContext>);\n} \n#endif \n\n###",560 "hip": " \n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/half_float_ops.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#ifdef CAFFE_HAS_HIP_FP16\nnamespace caffe2 {\nnamespace {\n__global__ void FloatToHalfKernel(const int N, const float* X, half* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = __float2half(X[i]);\n }\n}\n__global__ void HalfToFloatKernel(const int N, const half* X, float* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = __half2float(X[i]);\n }\n}\n}\ntemplate <>\nbool FloatToHalfOp<HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n auto* Y = Output(0, X.sizes(), at::dtype<at::Half>());\n hipLaunchKernelGGL(( FloatToHalfKernel), dim3(CAFFE_GET_BLOCKS(X.numel())), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), X.numel(), X.data<float>(), reinterpret_cast<half*>(Y->template mutable_data<at::Half>()));\n C10_HIP_KERNEL_LAUNCH_CHECK();\n return true;\n}\ntemplate <>\nbool HalfToFloatOp<HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( HalfToFloatKernel), dim3(CAFFE_GET_BLOCKS(X.numel())), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), X.numel(), reinterpret_cast<const half*>(X.data<at::Half>()), Y->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n return true;\n}\ntemplate <>\nbool Float16UniformFillOp<HIPContext>::RunOnDevice() {\n auto* output = Output(0, shape_, at::dtype<at::Half>());\n at::Half* out = output->template mutable_data<at::Half>();\n auto leading_dim_sz = output->size(0);\n CAFFE_ENFORCE_GT(leading_dim_sz, 0, \"The input shape should have the first dimension greater than 0\");\n int rowsz = output->numel() / output->size(0);\n ReinitializeTensor(\n &temp_data_buffer_, {rowsz}, at::dtype<float>().device(HIP));\n float* temp_data = temp_data_buffer_.template mutable_data<float>();\n for (uint64_t i = 0; i < leading_dim_sz; i++) {\n math::RandUniform<float, HIPContext>(\n rowsz, min_, max_, temp_data, &context_);\n hipLaunchKernelGGL(( FloatToHalfKernel), dim3(CAFFE_GET_BLOCKS(rowsz)), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), rowsz, temp_data, reinterpret_cast<half*>(out + i * rowsz));\n C10_HIP_KERNEL_LAUNCH_CHECK();\n }\n return true;\n}\nREGISTER_HIP_OPERATOR(FloatToHalf, FloatToHalfOp<HIPContext>);\nREGISTER_HIP_OPERATOR(HalfToFloat, HalfToFloatOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Float16UniformFill, Float16UniformFillOp<HIPContext>);\n} \n#endif ###"561 },562 {563 "cuda": "\n#include \"caffe2/operators/hard_sigmoid_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void HardSigmoidCUDAKernel(\n const int N,\n const T alpha,\n const T beta,\n const T* X,\n T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n Y[i] = max(T(0), min(T(1), alpha * __ldg(X + i) + beta));\n#else\n Y[i] = max(T(0), min(T(1), alpha * X[i] + beta));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void HardSigmoidGradientCUDAKernel(\n const int N,\n const T alpha,\n const T* dY,\n const T* Y,\n T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = (__ldg(Y + i) > T(0) && __ldg(Y + i) < T(1)) ? __ldg(dY + i) * alpha\n : T(0);\n#else\n dX[i] = (Y[i] > T(0) && Y[i] < T(1)) ? dY[i] * alpha : T(0);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool HardSigmoidFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n HardSigmoidCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, alpha, beta, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool HardSigmoidGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n HardSigmoidGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, alpha, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n HardSigmoid,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n HardSigmoidFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n HardSigmoidGradient,\n BinaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n HardSigmoidGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",564 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/hard_sigmoid_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void HardSigmoidHIPKernel(\n const int N,\n const T alpha,\n const T beta,\n const T* X,\n T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n Y[i] = max(T(0), min(T(1), alpha * __ldg(X + i) + beta));\n#else\n Y[i] = max(T(0), min(T(1), alpha * X[i] + beta));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void HardSigmoidGradientHIPKernel(\n const int N,\n const T alpha,\n const T* dY,\n const T* Y,\n T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = (__ldg(Y + i) > T(0) && __ldg(Y + i) < T(1)) ? __ldg(dY + i) * alpha\n : T(0);\n#else\n dX[i] = (Y[i] > T(0) && Y[i] < T(1)) ? dY[i] * alpha : T(0);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool HardSigmoidFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( HardSigmoidHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), N, alpha, beta, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool HardSigmoidGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( HardSigmoidGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, alpha, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n HardSigmoid,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n HardSigmoidFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n HardSigmoidGradient,\n BinaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n HardSigmoidGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"565 },566 {567 "cuda": "\n#include \"caffe2/operators/if_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(If, IfOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",568 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/if_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(If, IfOp<HIPContext>);\n\n} // namespace caffe2\n###"569 },570 {571 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/im2col_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Im2Col, Im2ColOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(Col2Im, Col2ImOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",572 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/im2col_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Im2Col, Im2ColOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(Col2Im, Col2ImOp<float, HIPContext>);\n\n} // namespace caffe2\n###"573 },574 {575 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/leaky_relu_op.h\"\n\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void LeakyReluKernel(const int N, const T alpha, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = X[i] >= 0 ? X[i] : X[i] * alpha;\n }\n}\n\ntemplate <typename T>\n__global__ void LeakyReluGradientKernel(\n const int N,\n const T alpha,\n const T* Y,\n const T* dY,\n T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n dX[i] = Y[i] >= 0 ? dY[i] : dY[i] * alpha;\n }\n}\n} // namespace\n\ntemplate <>\nbool LeakyReluOp<float, CUDAContext>::RunOnDevice() {\n const auto& X = Input(0);\n CAFFE_ENFORCE_GT(X.numel(), 0);\n\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n LeakyReluKernel<<<\n CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(), alpha_, X.data<float>(), Y->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool LeakyReluGradientOp<float, CUDAContext>::RunOnDevice() {\n const auto& Y = Input(0);\n const auto& dY = Input(1);\n\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n CAFFE_ENFORCE_EQ(Y.numel(), dY.numel());\n LeakyReluGradientKernel<<<\n CAFFE_GET_BLOCKS(Y.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n Y.numel(),\n alpha_,\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(LeakyRelu, LeakyReluOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n LeakyReluGradient,\n LeakyReluGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",576 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/leaky_relu_op.h\"\n\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void LeakyReluKernel(const int N, const T alpha, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = X[i] >= 0 ? X[i] : X[i] * alpha;\n }\n}\n\ntemplate <typename T>\n__global__ void LeakyReluGradientKernel(\n const int N,\n const T alpha,\n const T* Y,\n const T* dY,\n T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n dX[i] = Y[i] >= 0 ? dY[i] : dY[i] * alpha;\n }\n}\n} // namespace\n\ntemplate <>\nbool LeakyReluOp<float, HIPContext>::RunOnDevice() {\n const auto& X = Input(0);\n CAFFE_ENFORCE_GT(X.numel(), 0);\n\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( LeakyReluKernel), \n dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(), alpha_, X.data<float>(), Y->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool LeakyReluGradientOp<float, HIPContext>::RunOnDevice() {\n const auto& Y = Input(0);\n const auto& dY = Input(1);\n\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n CAFFE_ENFORCE_EQ(Y.numel(), dY.numel());\n hipLaunchKernelGGL(( LeakyReluGradientKernel), \n dim3(CAFFE_GET_BLOCKS(Y.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n Y.numel(),\n alpha_,\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(LeakyRelu, LeakyReluOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n LeakyReluGradient,\n LeakyReluGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"577 },578 {579 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\n\n// -----------------------------------\n// log_sigmoid forward\n// -----------------------------------\n\nvoid launch_log_sigmoid_forward_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, iter.common_dtype(), \"log_sigmoid_forward_cuda\", [&] {\n using opmath_t = at::opmath_type<scalar_t>;\n\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t in_) -> scalar_t {\n const opmath_t in = in_;\n const auto min = std::min(opmath_t(0), in);\n const auto z = std::exp(-std::abs(in));\n return min - std::log1p(z);\n });\n });\n}\n\nnamespace {\n// -----------------------------------\n// log_sigmoid backward\n// -----------------------------------\nvoid log_sigmoid_backward_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, iter.common_dtype(), \"log_sigmoid_backward_cuda\", [&] {\n using opmath_t = at::opmath_type<scalar_t>;\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t in_, scalar_t grad_out_) -> scalar_t {\n const opmath_t in = in_;\n const opmath_t grad_out = grad_out_;\n\n auto in_negative = in < opmath_t(0);\n auto max_deriv = in_negative ? opmath_t(1) : opmath_t(0);\n auto sign = in_negative ? opmath_t(1) : -opmath_t(1);\n const auto z = std::exp(-std::abs(in));\n return grad_out * (max_deriv - sign * (z / (opmath_t(1) + z)));\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(log_sigmoid_backward_stub, &log_sigmoid_backward_kernel);\n\n} // namespace at::native\n\n\n###",580 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\n\n// -----------------------------------\n// log_sigmoid forward\n// -----------------------------------\n\nvoid launch_log_sigmoid_forward_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, iter.common_dtype(), \"log_sigmoid_forward_hip\", [&] {\n using opmath_t = at::opmath_type<scalar_t>;\n\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t in_) -> scalar_t {\n const opmath_t in = in_;\n const auto min = ::min(opmath_t(0), in);\n const auto z = ::exp(-std::abs(in));\n return min - std::log1p(z);\n });\n });\n}\n\nnamespace {\n// -----------------------------------\n// log_sigmoid backward\n// -----------------------------------\nvoid log_sigmoid_backward_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, iter.common_dtype(), \"log_sigmoid_backward_hip\", [&] {\n using opmath_t = at::opmath_type<scalar_t>;\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t in_, scalar_t grad_out_) -> scalar_t {\n const opmath_t in = in_;\n const opmath_t grad_out = grad_out_;\n\n auto in_negative = in < opmath_t(0);\n auto max_deriv = in_negative ? opmath_t(1) : opmath_t(0);\n auto sign = in_negative ? opmath_t(1) : -opmath_t(1);\n const auto z = ::exp(-std::abs(in));\n return grad_out * (max_deriv - sign * (z / (opmath_t(1) + z)));\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(log_sigmoid_backward_stub, &log_sigmoid_backward_kernel);\n\n} // namespace at::native\n###"581 },582 {583 "cuda": "\n#include \"caffe2/operators/lengths_pad_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(LengthsPad, LengthsPadOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",584 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/lengths_pad_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(LengthsPad, LengthsPadOp<HIPContext>);\n} // namespace caffe2\n###"585 },586 {587 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/load_save_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nvoid LoadOp<CUDAContext>::SetCurrentDevice(BlobProto* proto) {\n if (proto->has_tensor()) {\n proto->mutable_tensor()->clear_device_detail();\n auto* device_detail = proto->mutable_tensor()->mutable_device_detail();\n device_detail->set_device_type(PROTO_CUDA);\n device_detail->set_device_id(CaffeCudaGetDevice());\n }\n}\n\nREGISTER_CUDA_OPERATOR(Load, LoadOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Save, SaveOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(Checkpoint, CheckpointOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",588 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/load_save_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nvoid LoadOp<HIPContext>::SetCurrentDevice(BlobProto* proto) {\n if (proto->has_tensor()) {\n proto->mutable_tensor()->clear_device_detail();\n auto* device_detail = proto->mutable_tensor()->mutable_device_detail();\n device_detail->set_device_type(PROTO_HIP);\n device_detail->set_device_id(CaffeHipGetDevice());\n }\n}\n\nREGISTER_HIP_OPERATOR(Load, LoadOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Save, SaveOp<HIPContext>);\nREGISTER_HIP_OPERATOR(Checkpoint, CheckpointOp<HIPContext>);\n} // namespace caffe2\n###"589 },590 {591 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/locally_connected_op.h\"\n#include \"caffe2/operators/locally_connected_op_impl.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(LC, LocallyConnectedOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n LCGradient,\n LocallyConnectedGradientOp<float, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(LC1D, LocallyConnectedOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n LC1DGradient,\n LocallyConnectedGradientOp<float, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(LC2D, LocallyConnectedOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n LC2DGradient,\n LocallyConnectedGradientOp<float, CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(LC3D, LocallyConnectedOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n LC3DGradient,\n LocallyConnectedGradientOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",592 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/locally_connected_op.h\"\n#include \"caffe2/operators/locally_connected_op_impl.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(LC, LocallyConnectedOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n LCGradient,\n LocallyConnectedGradientOp<float, HIPContext>);\n\nREGISTER_HIP_OPERATOR(LC1D, LocallyConnectedOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n LC1DGradient,\n LocallyConnectedGradientOp<float, HIPContext>);\n\nREGISTER_HIP_OPERATOR(LC2D, LocallyConnectedOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n LC2DGradient,\n LocallyConnectedGradientOp<float, HIPContext>);\n\nREGISTER_HIP_OPERATOR(LC3D, LocallyConnectedOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n LC3DGradient,\n LocallyConnectedGradientOp<float, HIPContext>);\n\n} // namespace caffe2\n###"593 },594 {595 "cuda": "\n#include \"caffe2/operators/log1p_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nLog1pGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / (__ldg(X + i) + T(1));\n#else\n dX[i] = dY[i] / (X[i] + T(1));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool Log1pGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n Log1pGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Log1p,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n Log1pFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n Log1pGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n Log1pGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",596 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/log1p_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nLog1pGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / (__ldg(X + i) + T(1));\n#else\n dX[i] = dY[i] / (X[i] + T(1));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool Log1pGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( Log1pGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Log1p,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n Log1pFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n Log1pGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n Log1pGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"597 },598 {599 "cuda": "\n#include \"caffe2/operators/logit_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void LogitKernel(const int N, const T* X, const float eps, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = fminf(X[i], (T(1) - eps));\n Y[i] = fmaxf(Y[i], eps);\n Y[i] = logf(Y[i] / (T(1) - Y[i]));\n }\n}\n\ntemplate <typename T>\n__global__ void LogitGradientKernel(\n const int N,\n const T* X,\n const T* dY,\n const float eps,\n T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n dX[i] = (X[i] < eps || X[i] > T(1) - eps) ? T(0)\n : (dY[i] / X[i] / (T(1) - X[i]));\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool LogitFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n LogitKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, X, eps_, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool LogitGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n auto& dY = Input(1);\n auto* dX = Output(0);\n dX->ResizeLike(X);\n int n = X.size();\n LogitGradientKernel<<<\n CAFFE_GET_BLOCKS(n),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n n,\n X.data<float>(),\n dY.data<float>(),\n eps_,\n dX->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Logit,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n LogitFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(LogitGradient, LogitGradientOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",600 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/logit_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void LogitKernel(const int N, const T* X, const float eps, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = fminf(X[i], (T(1) - eps));\n Y[i] = fmaxf(Y[i], eps);\n Y[i] = logf(Y[i] / (T(1) - Y[i]));\n }\n}\n\ntemplate <typename T>\n__global__ void LogitGradientKernel(\n const int N,\n const T* X,\n const T* dY,\n const float eps,\n T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n dX[i] = (X[i] < eps || X[i] > T(1) - eps) ? T(0)\n : (dY[i] / X[i] / (T(1) - X[i]));\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool LogitFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( LogitKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), N, X, eps_, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool LogitGradientOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n auto& dY = Input(1);\n auto* dX = Output(0);\n dX->ResizeLike(X);\n int n = X.size();\n hipLaunchKernelGGL(( LogitGradientKernel), \n dim3(CAFFE_GET_BLOCKS(n)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n n,\n X.data<float>(),\n dY.data<float>(),\n eps_,\n dX->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Logit,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n LogitFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(LogitGradient, LogitGradientOp<float, HIPContext>);\n\n} // namespace caffe2\n###"601 },602 {603 "cuda": "\n#include \"caffe2/operators/log_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Log,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n LogFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",604 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/log_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Log,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n LogFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"605 },606 {607 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/loss_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(AveragedLoss, AveragedLoss<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n AveragedLossGradient,\n AveragedLossGradient<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",608 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/loss_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(AveragedLoss, AveragedLoss<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n AveragedLossGradient,\n AveragedLossGradient<float, HIPContext>);\n} // namespace caffe2\n###"609 },610 {611 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/lpnorm_op.h\"\n#include \"caffe2/operators/operator_fallback_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(LpNorm, GPUFallbackOp);\nREGISTER_CUDA_OPERATOR(LpNormGradient, GPUFallbackOp);\n\n} // namespace caffe2\n\n\n###",612 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/lpnorm_op.h\"\n#include \"caffe2/operators/hip/operator_fallback_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(LpNorm, GPUFallbackOp);\nREGISTER_HIP_OPERATOR(LpNormGradient, GPUFallbackOp);\n\n} // namespace caffe2\n###"613 },614 {615 "cuda": "\n#include \"caffe2/operators/matmul_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(MatMul, MatMulOp<float, CUDAContext>);\n\n}\n\n\n###",616 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/matmul_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(MatMul, MatMulOp<float, HIPContext>);\n\n}\n###"617 },618 {619 "cuda": "\n#pragma once\n\n#include <cfloat>\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/logging.h\"\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/operators/conv_pool_op_base.h\"\n#include \"caffe2/operators/pool_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nclass MaxPoolWithIndexOp final : public ConvPoolOpBase<CUDAContext> {\n public:\n USE_CONV_POOL_BASE_FUNCTIONS(CUDAContext);\n MaxPoolWithIndexOp(const OperatorDef& operator_def, Workspace* ws)\n : ConvPoolOpBase<CUDAContext>(operator_def, ws) {}\n ~MaxPoolWithIndexOp() {}\n\n template <typename T>\n bool DoRunWithType();\n\n bool RunOnDevice() override;\n\n // Input: X\n // Output: Y, mask\n};\n\nclass MaxPoolWithIndexGradientOp final : public ConvPoolOpBase<CUDAContext> {\n public:\n USE_CONV_POOL_BASE_FUNCTIONS(CUDAContext);\n MaxPoolWithIndexGradientOp(const OperatorDef& operator_def, Workspace* ws)\n : ConvPoolOpBase<CUDAContext>(operator_def, ws) {}\n ~MaxPoolWithIndexGradientOp() {}\n\n template <typename T>\n bool DoRunWithType();\n\n bool RunOnDevice() override;\n\n // Input: X, dY, mask\n // Output: dX\n};\n\n}; // namespace caffe2\n\n\n###",620 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <cfloat>\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/logging.h\"\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/operators/conv_pool_op_base.h\"\n#include \"caffe2/operators/pool_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nclass MaxPoolWithIndexOp final : public ConvPoolOpBase<HIPContext> {\n public:\n USE_CONV_POOL_BASE_FUNCTIONS(HIPContext);\n MaxPoolWithIndexOp(const OperatorDef& operator_def, Workspace* ws)\n : ConvPoolOpBase<HIPContext>(operator_def, ws) {}\n ~MaxPoolWithIndexOp() {}\n\n template <typename T>\n bool DoRunWithType();\n\n bool RunOnDevice() override;\n\n // Input: X\n // Output: Y, mask\n};\n\nclass MaxPoolWithIndexGradientOp final : public ConvPoolOpBase<HIPContext> {\n public:\n USE_CONV_POOL_BASE_FUNCTIONS(HIPContext);\n MaxPoolWithIndexGradientOp(const OperatorDef& operator_def, Workspace* ws)\n : ConvPoolOpBase<HIPContext>(operator_def, ws) {}\n ~MaxPoolWithIndexGradientOp() {}\n\n template <typename T>\n bool DoRunWithType();\n\n bool RunOnDevice() override;\n\n // Input: X, dY, mask\n // Output: dX\n};\n\n}; // namespace caffe2\n###"621 },622 {623 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid mish_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"mish_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t x_acc = static_cast<opmath_t>(x);\n return x_acc *\n c10::cuda::compat::tanh(\n c10::cuda::compat::log1p(c10::cuda::compat::exp(x_acc)));\n });\n });\n}\n\nvoid mish_backward_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"mish_backward_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t dy, scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t dy_acc = static_cast<opmath_t>(dy);\n const opmath_t x_acc = static_cast<opmath_t>(x);\n const opmath_t s_acc =\n opmath_t(1) / (opmath_t(1) + c10::cuda::compat::exp(-x_acc));\n const opmath_t t_acc = c10::cuda::compat::tanh(\n c10::cuda::compat::log1p(c10::cuda::compat::exp(x_acc)));\n return dy_acc *\n (t_acc + x_acc * s_acc * (opmath_t(1) - t_acc * t_acc));\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(mish_stub, &mish_kernel);\nREGISTER_DISPATCH(mish_backward_stub, &mish_backward_kernel);\n\n} // namespace at::native\n\n\n###",624 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid mish_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"mish_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t x_acc = static_cast<opmath_t>(x);\n return x_acc *\n c10::hip::compat::tanh(\n c10::hip::compat::log1p(c10::hip::compat::exp(x_acc)));\n });\n });\n}\n\nvoid mish_backward_kernel(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"mish_backward_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t dy, scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t dy_acc = static_cast<opmath_t>(dy);\n const opmath_t x_acc = static_cast<opmath_t>(x);\n const opmath_t s_acc =\n opmath_t(1) / (opmath_t(1) + c10::hip::compat::exp(-x_acc));\n const opmath_t t_acc = c10::hip::compat::tanh(\n c10::hip::compat::log1p(c10::hip::compat::exp(x_acc)));\n return dy_acc *\n (t_acc + x_acc * s_acc * (opmath_t(1) - t_acc * t_acc));\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(mish_stub, &mish_kernel);\nREGISTER_DISPATCH(mish_backward_stub, &mish_backward_kernel);\n\n} // namespace at::native\n###"625 },626 {627 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/mean_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Mean, MeanOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(MeanGradient, MeanGradientOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",628 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/mean_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Mean, MeanOp<HIPContext>);\nREGISTER_HIP_OPERATOR(MeanGradient, MeanGradientOp<HIPContext>);\n\n} // namespace caffe2\n###"629 },630 {631 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/operator.h\"\n\nnamespace caffe2 {\nnamespace {\n\nclass GetGPUMemoryUsageOp final : public Operator<CUDAContext> {\n public:\n template<class... Args> explicit GetGPUMemoryUsageOp(Args&&... args)\n : Operator<CUDAContext>(std::forward<Args>(args)...) {}\n ~GetGPUMemoryUsageOp() override {}\n\n bool RunOnDevice() override {\n TORCH_CHECK_EQ(InputSize(), 0);\n TORCH_CHECK_EQ(OutputSize(), 1);\n std::vector<long> total_by_gpu = CUDAContext::TotalMemoryByGpu();\n std::vector<long> max_by_gpu = CUDAContext::MaxMemoryByGpu();\n TORCH_CHECK_EQ(total_by_gpu.size(), max_by_gpu.size());\n\n\n auto* stats = Output(0, {2, static_cast<int64_t>(total_by_gpu.size())}, at::dtype<long>());\n context_.CopyFromCPU<long>(\n total_by_gpu.size(),\n total_by_gpu.data(),\n stats->template mutable_data<long>());\n context_.CopyFromCPU<long>(\n max_by_gpu.size(),\n max_by_gpu.data(),\n stats->template mutable_data<long>() + total_by_gpu.size());\n return true;\n }\n};\n\nOPERATOR_SCHEMA(GetGPUMemoryUsage)\n .NumInputs(0)\n .NumOutputs(1)\n .SetDoc(R\"DOC(Fetches GPU memory stats from CUDAContext. Result is stored\n in output blob with shape (2, num_gpus). First row contains the total\n current memory usage, and the second row the maximum usage during\n this execution.\n\n NOTE: --caffe2_gpu_memory_tracking flag must be enabled to use this op.\n )DOC\");\n\nREGISTER_CUDA_OPERATOR(GetGPUMemoryUsage, GetGPUMemoryUsageOp);\n}\n\n} // namespace caffe2\n\n\n###",632 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/operator.h\"\n\nnamespace caffe2 {\nnamespace {\n\nclass GetGPUMemoryUsageOp final : public Operator<HIPContext> {\n public:\n template<class... Args> explicit GetGPUMemoryUsageOp(Args&&... args)\n : Operator<HIPContext>(std::forward<Args>(args)...) {}\n ~GetGPUMemoryUsageOp() override {}\n\n bool RunOnDevice() override {\n TORCH_CHECK_EQ(InputSize(), 0);\n TORCH_CHECK_EQ(OutputSize(), 1);\n std::vector<long> total_by_gpu = HIPContext::TotalMemoryByGpu();\n std::vector<long> max_by_gpu = HIPContext::MaxMemoryByGpu();\n TORCH_CHECK_EQ(total_by_gpu.size(), max_by_gpu.size());\n\n\n auto* stats = Output(0, {2, static_cast<int64_t>(total_by_gpu.size())}, at::dtype<long>());\n context_.CopyFromCPU<long>(\n total_by_gpu.size(),\n total_by_gpu.data(),\n stats->template mutable_data<long>());\n context_.CopyFromCPU<long>(\n max_by_gpu.size(),\n max_by_gpu.data(),\n stats->template mutable_data<long>() + total_by_gpu.size());\n return true;\n }\n};\n\nOPERATOR_SCHEMA(GetGPUMemoryUsage)\n .NumInputs(0)\n .NumOutputs(1)\n .SetDoc(R\"DOC(Fetches GPU memory stats from HIPContext. Result is stored\n in output blob with shape (2, num_gpus). First row contains the total\n current memory usage, and the second row the maximum usage during\n this execution.\n\n NOTE: --caffe2_gpu_memory_tracking flag must be enabled to use this op.\n )DOC\");\n\nREGISTER_HIP_OPERATOR(GetGPUMemoryUsage, GetGPUMemoryUsageOp);\n}\n\n} // namespace caffe2\n###"633 },634 {635 "cuda": "\n#include \"caffe2/operators/minmax_ops.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void SelectGradientCUDAKernel(\n const int N,\n const T* dY,\n const T* X,\n const T* Y,\n T* dX) {\n const int i = blockIdx.x * CAFFE_CUDA_NUM_THREADS + threadIdx.x;\n if (i < N) {\n#if __CUDA_ARCH__ >= 350 || defined(USE_ROCM)\n dX[i] = __ldg(X + i) == __ldg(Y + i) ? __ldg(dY + i) : T(0);\n#else\n dX[i] = X[i] == Y[i] ? dY[i] : T(0);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\nbool SelectGradientOpBase<float, CUDAContext>::RunOnDevice() {\n const auto& Y = Input(0);\n const auto& dY = Input(1);\n const int N = Y.numel();\n const int M = math::DivUp(N, CAFFE_CUDA_NUM_THREADS);\n const float* dY_data = dY.data<float>();\n const float* Y_data = Y.data<float>();\n for (int i = 0; i < OutputSize(); i++) {\n const auto& Xi = Input(i + 2);\n auto* dXi = Output(i, Xi.sizes(), at::dtype<float>());\n const float* Xi_data = Xi.data<float>();\n float* dXi_data = dXi->mutable_data<float>();\n if (N > 0) {\n SelectGradientCUDAKernel<float>\n <<<M, CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n N, dY_data, Xi_data, Y_data, dXi_data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n }\n }\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Min, MinOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(MinGradient, MinGradientOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(Max, MaxOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(MaxGradient, MaxGradientOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",636 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/minmax_ops.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void SelectGradientHIPKernel(\n const int N,\n const T* dY,\n const T* X,\n const T* Y,\n T* dX) {\n const int i = blockIdx.x * CAFFE_HIP_NUM_THREADS + threadIdx.x;\n if (i < N) {\n#if __HIP_ARCH__ >= 350 || defined(USE_ROCM)\n dX[i] = __ldg(X + i) == __ldg(Y + i) ? __ldg(dY + i) : T(0);\n#else\n dX[i] = X[i] == Y[i] ? dY[i] : T(0);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\nbool SelectGradientOpBase<float, HIPContext>::RunOnDevice() {\n const auto& Y = Input(0);\n const auto& dY = Input(1);\n const int N = Y.numel();\n const int M = math::DivUp(N, CAFFE_HIP_NUM_THREADS);\n const float* dY_data = dY.data<float>();\n const float* Y_data = Y.data<float>();\n for (int i = 0; i < OutputSize(); i++) {\n const auto& Xi = Input(i + 2);\n auto* dXi = Output(i, Xi.sizes(), at::dtype<float>());\n const float* Xi_data = Xi.data<float>();\n float* dXi_data = dXi->mutable_data<float>();\n if (N > 0) {\n hipLaunchKernelGGL(( SelectGradientHIPKernel<float>)\n , dim3(M), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), \n N, dY_data, Xi_data, Y_data, dXi_data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n }\n }\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Min, MinOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(MinGradient, MinGradientOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(Max, MaxOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(MaxGradient, MaxGradientOp<float, HIPContext>);\n\n} // namespace caffe2\n###"637 },638 {639 "cuda": "\n#include \"caffe2/operators/mod_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void ModOpSimpleKernel(const int N, const int64_t divisor_,\n const T* data_ptr, T* output_ptr) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n output_ptr[i] = data_ptr[i] % divisor_;\n }\n}\n\n\ntemplate <typename T>\n__global__ void ModOpKernel(const int N, const int64_t divisor_,\n const T* data_ptr, T* output_ptr) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n output_ptr[i] = data_ptr[i] % divisor_;\n if (output_ptr[i] && ((output_ptr[i] > 0) != (divisor_ > 0))) {\n output_ptr[i] += divisor_;\n }\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ModOp<CUDAContext>::DoRunWithType() {\n auto& data = Input(DATA);\n auto N = data.numel();\n const auto* data_ptr = data.template data<T>();\n\n auto* output = Output(0, data.sizes(), at::dtype<T>());\n auto* output_ptr = output->template mutable_data<T>();\n\n if (sign_follow_divisor_) {\n ModOpKernel<<<\n CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n N, divisor_, data_ptr, output_ptr);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n } else {\n ModOpSimpleKernel<<<\n CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n N, divisor_, data_ptr, output_ptr);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n }\n\n return true;\n\n}\n\nREGISTER_CUDA_OPERATOR(Mod, ModOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",640 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/mod_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void ModOpSimpleKernel(const int N, const int64_t divisor_,\n const T* data_ptr, T* output_ptr) {\n HIP_1D_KERNEL_LOOP(i, N) {\n output_ptr[i] = data_ptr[i] % divisor_;\n }\n}\n\n\ntemplate <typename T>\n__global__ void ModOpKernel(const int N, const int64_t divisor_,\n const T* data_ptr, T* output_ptr) {\n HIP_1D_KERNEL_LOOP(i, N) {\n output_ptr[i] = data_ptr[i] % divisor_;\n if (output_ptr[i] && ((output_ptr[i] > 0) != (divisor_ > 0))) {\n output_ptr[i] += divisor_;\n }\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ModOp<HIPContext>::DoRunWithType() {\n auto& data = Input(DATA);\n auto N = data.numel();\n const auto* data_ptr = data.template data<T>();\n\n auto* output = Output(0, data.sizes(), at::dtype<T>());\n auto* output_ptr = output->template mutable_data<T>();\n\n if (sign_follow_divisor_) {\n hipLaunchKernelGGL(( ModOpKernel), \n dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n N, divisor_, data_ptr, output_ptr);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n } else {\n hipLaunchKernelGGL(( ModOpSimpleKernel), \n dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n N, divisor_, data_ptr, output_ptr);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n }\n\n return true;\n\n}\n\nREGISTER_HIP_OPERATOR(Mod, ModOp<HIPContext>);\n\n} // namespace caffe2\n###"641 },642 {643 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/multi_class_accuracy_op.h\"\n#include \"caffe2/utils/GpuAtomics.cuh\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void MultiClassAccuracyKernel(const int N, const int D, const float* Xdata,\n const int* labeldata, float* accuracies, int* amounts) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n float maxval = Xdata[i * D];\n int maxid = 0;\n for (int j = 1; j < D; ++j) {\n if (Xdata[i * D + j] > maxval) {\n maxval = Xdata[i * D + j];\n maxid = j;\n }\n }\n int labelid = labeldata[i];\n if (maxid == labelid) {\n gpu_atomic_add(accuracies + labelid, static_cast<float>(1));\n }\n gpu_atomic_add(amounts + labelid, static_cast<int>(1));\n }\n}\n__global__ void MultiClassAccuracyDivideKernel(\n const int D, float* accuracies, const int* amounts) {\n CUDA_1D_KERNEL_LOOP(i, D) {\n if (amounts[i]) {\n accuracies[i] /= amounts[i];\n }\n }\n}\n} // namespace\n\ntemplate <>\nbool MultiClassAccuracyOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(PREDICTION);\n auto& label = Input(LABEL);\n\n\n TORCH_DCHECK_EQ(X.dim(), 2);\n // amount, number of instances\n int N = X.dim32(0);\n // dimension, number of classes\n int D = X.dim32(1);\n TORCH_DCHECK_EQ(label.dim(), 1);\n TORCH_DCHECK_EQ(label.dim32(0), N);\n auto* Y0 = Output(0, {D}, at::dtype<float>());\n auto* Y1 = Output(1, {D}, at::dtype<int>());\n\n const float* Xdata = X.data<float>();\n const int* labeldata = label.data<int>();\n float* accuracies = Y0->template mutable_data<float>();\n int* amounts = Y1->template mutable_data<int>();\n math::Set<float, CUDAContext>(D, 0.0, accuracies, &context_);\n math::Set<int, CUDAContext>(D, 0, amounts, &context_);\n\n MultiClassAccuracyKernel<<<CAFFE_GET_BLOCKS(N), CAFFE_CUDA_NUM_THREADS,\n 0, context_.cuda_stream()>>>(\n N, D, Xdata, labeldata, accuracies, amounts);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n MultiClassAccuracyDivideKernel<<<CAFFE_GET_BLOCKS(D), CAFFE_CUDA_NUM_THREADS,\n 0, context_.cuda_stream()>>>(\n D, accuracies, amounts);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n MultiClassAccuracy, MultiClassAccuracyOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",644 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/multi_class_accuracy_op.h\"\n#include \"caffe2/utils/hip/GpuAtomics.cuh\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nnamespace {\n__global__ void MultiClassAccuracyKernel(const int N, const int D, const float* Xdata,\n const int* labeldata, float* accuracies, int* amounts) {\n HIP_1D_KERNEL_LOOP(i, N) {\n float maxval = Xdata[i * D];\n int maxid = 0;\n for (int j = 1; j < D; ++j) {\n if (Xdata[i * D + j] > maxval) {\n maxval = Xdata[i * D + j];\n maxid = j;\n }\n }\n int labelid = labeldata[i];\n if (maxid == labelid) {\n gpu_atomic_add(accuracies + labelid, static_cast<float>(1));\n }\n gpu_atomic_add(amounts + labelid, static_cast<int>(1));\n }\n}\n__global__ void MultiClassAccuracyDivideKernel(\n const int D, float* accuracies, const int* amounts) {\n HIP_1D_KERNEL_LOOP(i, D) {\n if (amounts[i]) {\n accuracies[i] /= amounts[i];\n }\n }\n}\n} // namespace\n\ntemplate <>\nbool MultiClassAccuracyOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(PREDICTION);\n auto& label = Input(LABEL);\n\n\n TORCH_DCHECK_EQ(X.dim(), 2);\n // amount, number of instances\n int N = X.dim32(0);\n // dimension, number of classes\n int D = X.dim32(1);\n TORCH_DCHECK_EQ(label.dim(), 1);\n TORCH_DCHECK_EQ(label.dim32(0), N);\n auto* Y0 = Output(0, {D}, at::dtype<float>());\n auto* Y1 = Output(1, {D}, at::dtype<int>());\n\n const float* Xdata = X.data<float>();\n const int* labeldata = label.data<int>();\n float* accuracies = Y0->template mutable_data<float>();\n int* amounts = Y1->template mutable_data<int>();\n math::Set<float, HIPContext>(D, 0.0, accuracies, &context_);\n math::Set<int, HIPContext>(D, 0, amounts, &context_);\n\n hipLaunchKernelGGL(( MultiClassAccuracyKernel), dim3(CAFFE_GET_BLOCKS(N)), dim3(CAFFE_HIP_NUM_THREADS),\n 0, context_.hip_stream(), \n N, D, Xdata, labeldata, accuracies, amounts);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n hipLaunchKernelGGL(( MultiClassAccuracyDivideKernel), dim3(CAFFE_GET_BLOCKS(D)), dim3(CAFFE_HIP_NUM_THREADS),\n 0, context_.hip_stream(), \n D, accuracies, amounts);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n MultiClassAccuracy, MultiClassAccuracyOp<float, HIPContext>);\n} // namespace caffe2\n###"645 },646 {647 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/negate_gradient_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(NegateGradient, NegateGradientOp<CUDAContext>)\n} // namespace caffe2\n\n\n###",648 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/negate_gradient_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(NegateGradient, NegateGradientOp<HIPContext>)\n} // namespace caffe2\n###"649 },650 {651 "cuda": "\n#include \"caffe2/operators/negative_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Negative,\n UnaryElementwiseOp<\n NumericTypes,\n CUDAContext,\n NegativeFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",652 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/negative_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Negative,\n UnaryElementwiseOp<\n NumericTypes,\n HIPContext,\n NegativeFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"653 },654 {655 "cuda": "\n#include <cub/block/block_reduce.cuh>\n\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/one_hot_ops.h\"\n#include \"caffe2/utils/cub_namespace.cuh\"\n\nnamespace caffe2 {\n\n__global__ void OneHotOpKernel(\n const int64_t batch_size,\n const int64_t index_size,\n const int64_t* indices,\n float* output) {\n CUDA_1D_KERNEL_LOOP(i, batch_size) {\n output[i * index_size + indices[i]] = 1.;\n }\n}\n\ntemplate <>\nvoid OneHotOp<CUDAContext>::DoOneHotOp(\n int64_t batch_size,\n int64_t index_size,\n const Tensor& indices,\n Tensor* output) {\n float* output_ptr = output->template mutable_data<float>();\n math::Set<float, CUDAContext>(output->numel(), 0., output_ptr, &context_);\n OneHotOpKernel<<<\n CAFFE_GET_BLOCKS(batch_size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n batch_size, index_size, indices.data<int64_t>(), output_ptr);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_CUDA_OPERATOR(OneHot, OneHotOp<CUDAContext>);\n} // namespace\n\n\n###",656 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include <hipcub/hipcub.hpp>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/one_hot_ops.h\"\n#include \"caffe2/utils/cub_namespace.cuh\"\n\nnamespace caffe2 {\n\n__global__ void OneHotOpKernel(\n const int64_t batch_size,\n const int64_t index_size,\n const int64_t* indices,\n float* output) {\n HIP_1D_KERNEL_LOOP(i, batch_size) {\n output[i * index_size + indices[i]] = 1.;\n }\n}\n\ntemplate <>\nvoid OneHotOp<HIPContext>::DoOneHotOp(\n int64_t batch_size,\n int64_t index_size,\n const Tensor& indices,\n Tensor* output) {\n float* output_ptr = output->template mutable_data<float>();\n math::Set<float, HIPContext>(output->numel(), 0., output_ptr, &context_);\n hipLaunchKernelGGL(( OneHotOpKernel), \n dim3(CAFFE_GET_BLOCKS(batch_size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n batch_size, index_size, indices.data<int64_t>(), output_ptr);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_HIP_OPERATOR(OneHot, OneHotOp<HIPContext>);\n} // namespace\n###"657 },658 {659 "cuda": "\n#ifndef CAFFE2_OPERATORS_OPERATOR_FALLBACK_H_\n#define CAFFE2_OPERATORS_OPERATOR_FALLBACK_H_\n#include \"caffe2/core/common.h\"\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/proto/caffe2_pb.h\"\nnamespace caffe2 {\n\ntemplate <typename SkipOutputCopy>\nclass GPUFallbackOpEx final : public Operator<CUDAContext> {\n public:\n USE_OPERATOR_FUNCTIONS(CUDAContext);\n explicit GPUFallbackOpEx(const OperatorDef& def, Workspace* ws)\n : Operator<CUDAContext>(def, ws) {\n CAFFE_ENFORCE_EQ(def.device_option().device_type(), PROTO_CUDA);\n OperatorDef base_def_(def);\n \n base_def_.clear_device_option();\n base_def_.mutable_device_option()->set_device_type(PROTO_CPU);\n \n for (const string& name : def.input()) {\n local_input_blobs_.push_back(local_ws_.CreateBlob(name));\n TORCH_CHECK_NOTNULL(local_input_blobs_.back());\n }\n base_op_ = CreateOperator(base_def_, &local_ws_);\n for (const string& name : def.output()) {\n local_output_blobs_.push_back(local_ws_.GetBlob(name));\n TORCH_CHECK_NOTNULL(local_output_blobs_.back());\n }\n }\n bool RunOnDevice() override {\n for (const auto i : c10::irange(InputSize())) {\n if (this->InputIsTensorType(i, CUDA)) {\n \n BlobGetMutableTensor(local_input_blobs_[i], CPU)->CopyFrom(Input(i));\n } else {\n VLOG(1) << \"Input \" << i << \" is not TensorCUDA. Skipping copy.\";\n \n \n \n local_input_blobs_[i]->ShareExternal(\n const_cast<void*>(OperatorBase::Inputs()[i]->GetRaw()), OperatorBase::Inputs()[i]->meta());\n }\n }\n if (!base_op_->Run()) {\n LOG(ERROR) << \"Base op run failed in GPUFallbackOp. Def: \"\n << ProtoDebugString(this->debug_def());\n return false;\n }\n for (const auto i : c10::irange(OutputSize())) {\n if (SkipOutputCopy::Contains(i)) {\n VLOG(1) << \"Copy output: index \" << i << \" skipped.\";\n continue;\n }\n CAFFE_ENFORCE(\n BlobIsTensorType(*local_output_blobs_[i], CPU), \"GPU fallback op currently does not support non-TensorCPU \"\n \"output type who needs copying.\");\n Output(i)->CopyFrom(local_output_blobs_[i]->template Get<TensorCPU>());\n }\n return true;\n }\n protected:\n Workspace local_ws_;\n vector<Blob*> local_input_blobs_;\n vector<Blob*> local_output_blobs_;\n unique_ptr<OperatorBase> base_op_;\n};\nusing GPUFallbackOp = GPUFallbackOpEx<SkipIndices<>>;\n} \n#endif \n\n###",660 "hip": " \n#ifndef CAFFE2_OPERATORS_OPERATOR_FALLBACK_H_\n#define CAFFE2_OPERATORS_OPERATOR_FALLBACK_H_\n#include \"caffe2/core/common.h\"\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/proto/caffe2_pb.h\"\nnamespace caffe2 {\n\ntemplate <typename SkipOutputCopy>\nclass GPUFallbackOpEx final : public Operator<HIPContext> {\n public:\n USE_OPERATOR_FUNCTIONS(HIPContext);\n explicit GPUFallbackOpEx(const OperatorDef& def, Workspace* ws)\n : Operator<HIPContext>(def, ws) {\n CAFFE_ENFORCE_EQ(def.device_option().device_type(), PROTO_HIP);\n OperatorDef base_def_(def);\n \n base_def_.clear_device_option();\n base_def_.mutable_device_option()->set_device_type(PROTO_CPU);\n \n for (const string& name : def.input()) {\n local_input_blobs_.push_back(local_ws_.CreateBlob(name));\n TORCH_CHECK_NOTNULL(local_input_blobs_.back());\n }\n base_op_ = CreateOperator(base_def_, &local_ws_);\n for (const string& name : def.output()) {\n local_output_blobs_.push_back(local_ws_.GetBlob(name));\n TORCH_CHECK_NOTNULL(local_output_blobs_.back());\n }\n }\n bool RunOnDevice() override {\n for (const auto i : c10::irange(InputSize())) {\n if (this->InputIsTensorType(i, HIP)) {\n \n BlobGetMutableTensor(local_input_blobs_[i], CPU)->CopyFrom(Input(i));\n } else {\n VLOG(1) << \"Input \" << i << \" is not TensorHIP. Skipping copy.\";\n \n \n \n local_input_blobs_[i]->ShareExternal(\n const_cast<void*>(OperatorBase::Inputs()[i]->GetRaw()), OperatorBase::Inputs()[i]->meta());\n }\n }\n if (!base_op_->Run()) {\n LOG(ERROR) << \"Base op run failed in GPUFallbackOp. Def: \"\n << ProtoDebugString(this->debug_def());\n return false;\n }\n for (const auto i : c10::irange(OutputSize())) {\n if (SkipOutputCopy::Contains(i)) {\n VLOG(1) << \"Copy output: index \" << i << \" skipped.\";\n continue;\n }\n CAFFE_ENFORCE(\n BlobIsTensorType(*local_output_blobs_[i], CPU), \"GPU fallback op currently does not support non-TensorCPU \"\n \"output type who needs copying.\");\n Output(i)->CopyFrom(local_output_blobs_[i]->template Get<TensorCPU>());\n }\n return true;\n }\n protected:\n Workspace local_ws_;\n vector<Blob*> local_input_blobs_;\n vector<Blob*> local_output_blobs_;\n unique_ptr<OperatorBase> base_op_;\n};\nusing GPUFallbackOp = GPUFallbackOpEx<SkipIndices<>>;\n} \n#endif ###"661 },662 {663 "cuda": "\n#include <iostream>\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/operators/operator_fallback_gpu.h\"\n#include <gtest/gtest.h>\nnamespace caffe2 {\nclass IncrementByOneOp final : public Operator<CPUContext> {\n public:\n template <class... Args>\n explicit IncrementByOneOp(Args&&... args)\n : Operator<CPUContext>(std::forward<Args>(args)...) {}\n bool RunOnDevice() override {\n const auto& in = Input(0);\n auto* out = Output(0, in.sizes(), at::dtype<float>());\n const float* in_data = in.template data<float>();\n float* out_data = out->template mutable_data<float>();\n for (int i = 0; i < in.numel(); ++i) {\n out_data[i] = in_data[i] + 1.f;\n }\n return true;\n }\n};\nOPERATOR_SCHEMA(IncrementByOne)\n .NumInputs(1).NumOutputs(1).AllowInplace({{0, 0}});\nREGISTER_CPU_OPERATOR(IncrementByOne, IncrementByOneOp);\nREGISTER_CUDA_OPERATOR(IncrementByOne, GPUFallbackOp);\nTEST(OperatorFallbackTest, IncrementByOneOp) {\n OperatorDef op_def = CreateOperatorDef(\n \"IncrementByOne\", \"\", vector<string>{\"X\"}, vector<string>{\"X\"});\n Workspace ws;\n Tensor source_tensor(vector<int64_t>{2, 3}, CPU);\n for (int i = 0; i < 6; ++i) {\n source_tensor.mutable_data<float>()[i] = i;\n }\n BlobGetMutableTensor(ws.CreateBlob(\"X\"), CPU)->CopyFrom(source_tensor);\n unique_ptr<OperatorBase> op(CreateOperator(op_def, &ws));\n EXPECT_TRUE(op.get() != nullptr);\n EXPECT_TRUE(op->Run());\n const TensorCPU& output = ws.GetBlob(\"X\")->Get<TensorCPU>();\n EXPECT_EQ(output.dim(), 2);\n EXPECT_EQ(output.size(0), 2);\n EXPECT_EQ(output.size(1), 3);\n for (int i = 0; i < 6; ++i) {\n EXPECT_EQ(output.data<float>()[i], i + 1);\n }\n}\nTEST(OperatorFallbackTest, GPUIncrementByOneOp) {\n if (!HasCudaGPU()) return;\n OperatorDef op_def = CreateOperatorDef(\n \"IncrementByOne\", \"\", vector<string>{\"X\"}, vector<string>{\"X\"});\n op_def.mutable_device_option()->set_device_type(PROTO_CUDA);\n Workspace ws;\n Tensor source_tensor(vector<int64_t>{2, 3}, CPU);\n for (int i = 0; i < 6; ++i) {\n source_tensor.mutable_data<float>()[i] = i;\n }\n BlobGetMutableTensor(ws.CreateBlob(\"X\"), CUDA)->CopyFrom(source_tensor);\n unique_ptr<OperatorBase> op(CreateOperator(op_def, &ws));\n EXPECT_TRUE(op.get() != nullptr);\n EXPECT_TRUE(op->Run());\n const TensorCUDA& output = ws.GetBlob(\"X\")->Get<TensorCUDA>();\n Tensor output_cpu(output, CPU);\n EXPECT_EQ(output.dim(), 2);\n EXPECT_EQ(output.size(0), 2);\n EXPECT_EQ(output.size(1), 3);\n for (int i = 0; i < 6; ++i) {\n EXPECT_EQ(output_cpu.data<float>()[i], i + 1);\n }\n}\n} \n\n###",664 "hip": " \n#include <iostream>\n#include \"caffe2/core/operator.h\"\n#include \"caffe2/operators/hip/operator_fallback_gpu.h\"\n#include <gtest/gtest.h>\nnamespace caffe2 {\nclass IncrementByOneOp final : public Operator<CPUContext> {\n public:\n template <class... Args>\n explicit IncrementByOneOp(Args&&... args)\n : Operator<CPUContext>(std::forward<Args>(args)...) {}\n bool RunOnDevice() override {\n const auto& in = Input(0);\n auto* out = Output(0, in.sizes(), at::dtype<float>());\n const float* in_data = in.template data<float>();\n float* out_data = out->template mutable_data<float>();\n for (int i = 0; i < in.numel(); ++i) {\n out_data[i] = in_data[i] + 1.f;\n }\n return true;\n }\n};\nOPERATOR_SCHEMA(IncrementByOne)\n .NumInputs(1).NumOutputs(1).AllowInplace({{0, 0}});\nREGISTER_CPU_OPERATOR(IncrementByOne, IncrementByOneOp);\nREGISTER_HIP_OPERATOR(IncrementByOne, GPUFallbackOp);\nTEST(OperatorFallbackTest, IncrementByOneOp) {\n OperatorDef op_def = CreateOperatorDef(\n \"IncrementByOne\", \"\", vector<string>{\"X\"}, vector<string>{\"X\"});\n Workspace ws;\n Tensor source_tensor(vector<int64_t>{2, 3}, CPU);\n for (int i = 0; i < 6; ++i) {\n source_tensor.mutable_data<float>()[i] = i;\n }\n BlobGetMutableTensor(ws.CreateBlob(\"X\"), CPU)->CopyFrom(source_tensor);\n unique_ptr<OperatorBase> op(CreateOperator(op_def, &ws));\n EXPECT_TRUE(op.get() != nullptr);\n EXPECT_TRUE(op->Run());\n const TensorCPU& output = ws.GetBlob(\"X\")->Get<TensorCPU>();\n EXPECT_EQ(output.dim(), 2);\n EXPECT_EQ(output.size(0), 2);\n EXPECT_EQ(output.size(1), 3);\n for (int i = 0; i < 6; ++i) {\n EXPECT_EQ(output.data<float>()[i], i + 1);\n }\n}\nTEST(OperatorFallbackTest, GPUIncrementByOneOp) {\n if (!HasHipGPU()) return;\n OperatorDef op_def = CreateOperatorDef(\n \"IncrementByOne\", \"\", vector<string>{\"X\"}, vector<string>{\"X\"});\n op_def.mutable_device_option()->set_device_type(PROTO_HIP);\n Workspace ws;\n Tensor source_tensor(vector<int64_t>{2, 3}, CPU);\n for (int i = 0; i < 6; ++i) {\n source_tensor.mutable_data<float>()[i] = i;\n }\n BlobGetMutableTensor(ws.CreateBlob(\"X\"), HIP)->CopyFrom(source_tensor);\n unique_ptr<OperatorBase> op(CreateOperator(op_def, &ws));\n EXPECT_TRUE(op.get() != nullptr);\n EXPECT_TRUE(op->Run());\n const TensorHIP& output = ws.GetBlob(\"X\")->Get<TensorHIP>();\n Tensor output_cpu(output, CPU);\n EXPECT_EQ(output.dim(), 2);\n EXPECT_EQ(output.size(0), 2);\n EXPECT_EQ(output.size(1), 3);\n for (int i = 0; i < 6; ++i) {\n EXPECT_EQ(output_cpu.data<float>()[i], i + 1);\n }\n}\n} ###"665 },666 {667 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\n\n// -----------------------------------\n// prelu\n// -----------------------------------\nvoid prelu_kernel(TensorIterator &iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(kBFloat16, kHalf, iter.dtype(), \"prelu_cuda\", [&] {\n gpu_kernel(iter,\n [] GPU_LAMBDA (scalar_t input, scalar_t weight) -> scalar_t {\n return (input > 0) ? input : weight * input;\n });\n });\n}\n\nvoid prelu_backward_kernel(TensorIterator &iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(kBFloat16, kHalf, iter.dtype(), \"prelu_backward_cuda\", [&] {\n gpu_kernel_multiple_outputs(iter,\n [] GPU_LAMBDA (scalar_t input, scalar_t weight, scalar_t grad) -> thrust::tuple<scalar_t, scalar_t> {\n auto mask = input > 0;\n auto grad_input = mask ? grad : weight * grad;\n auto grad_weight = mask ? scalar_t{0} : input * grad;\n return {grad_input, grad_weight};\n });\n });\n}\n\nREGISTER_DISPATCH(prelu_stub, &prelu_kernel);\nREGISTER_DISPATCH(prelu_backward_stub, &prelu_backward_kernel);\n\n} // namespace at::native\n\n\n###",668 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\n\n// -----------------------------------\n// prelu\n// -----------------------------------\nvoid prelu_kernel(TensorIterator &iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(kBFloat16, kHalf, iter.dtype(), \"prelu_hip\", [&] {\n gpu_kernel(iter,\n [] GPU_LAMBDA (scalar_t input, scalar_t weight) -> scalar_t {\n return (input > 0) ? input : weight * input;\n });\n });\n}\n\nvoid prelu_backward_kernel(TensorIterator &iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(kBFloat16, kHalf, iter.dtype(), \"prelu_backward_hip\", [&] {\n gpu_kernel_multiple_outputs(iter,\n [] GPU_LAMBDA (scalar_t input, scalar_t weight, scalar_t grad) -> thrust::tuple<scalar_t, scalar_t> {\n auto mask = input > 0;\n auto grad_input = mask ? grad : weight * grad;\n auto grad_weight = mask ? scalar_t{0} : input * grad;\n return {grad_input, grad_weight};\n });\n });\n}\n\nREGISTER_DISPATCH(prelu_stub, &prelu_kernel);\nREGISTER_DISPATCH(prelu_backward_stub, &prelu_backward_kernel);\n\n} // namespace at::native\n###"669 },670 {671 "cuda": "\n#include \"caffe2/operators/order_switch_ops.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(NHWC2NCHW, NHWC2NCHWOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(NCHW2NHWC, NCHW2NHWCOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",672 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/order_switch_ops.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(NHWC2NCHW, NHWC2NCHWOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(NCHW2NHWC, NCHW2NHWCOp<float, HIPContext>);\n\n} // namespace caffe2\n###"673 },674 {675 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/perplexity_op.h\"\n#include \"caffe2/utils/math.h\"\n#include <thrust/device_vector.h>\n#include <thrust/transform_reduce.h>\n#include <thrust/system/cuda/execution_policy.h>\n\nnamespace caffe2 {\n\nstruct perplexity_function\n{\n perplexity_function(float p) : pow(p) {}\n __host__ __device__ float operator()(float x) const\n {\n return powf(1.0f/x, pow);\n }\n float pow;\n};\n\ntemplate <>\nbool PerplexityOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n TORCH_DCHECK_EQ(X.dim(), 1);\n int N = X.dim32(0);\n\n auto* Y = Output(0, vector<int64_t>(), at::dtype<float>());\n float* Ydata = Y->template mutable_data<float>();\n const float* Xdata = X.data<float>();\n\n float perplexity = thrust::transform_reduce(\n #if THRUST_VERSION >= 100800\n thrust::cuda::par.on(context_.cuda_stream()),\n #endif // THRUST_VERSION >= 100800\n Xdata, Xdata + N,\n perplexity_function(1.0f/N),\n 1.0f,\n thrust::multiplies<float>());\n\n math::Set<float, CUDAContext>(1, perplexity, Ydata, &context_);\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Perplexity, PerplexityOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",676 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/perplexity_op.h\"\n#include \"caffe2/utils/math.h\"\n#include <thrust/device_vector.h>\n#include <thrust/transform_reduce.h>\n#include <thrust/system/hip/execution_policy.h>\n\nnamespace caffe2 {\n\nstruct perplexity_function\n{\n perplexity_function(float p) : pow(p) {}\n __host__ __device__ float operator()(float x) const\n {\n return powf(1.0f/x, pow);\n }\n float pow;\n};\n\ntemplate <>\nbool PerplexityOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n TORCH_DCHECK_EQ(X.dim(), 1);\n int N = X.dim32(0);\n\n auto* Y = Output(0, vector<int64_t>(), at::dtype<float>());\n float* Ydata = Y->template mutable_data<float>();\n const float* Xdata = X.data<float>();\n\n float perplexity = thrust::transform_reduce(\n #if THRUST_VERSION >= 100800\n thrust::hip::par.on(context_.hip_stream()),\n #endif // THRUST_VERSION >= 100800\n Xdata, Xdata + N,\n perplexity_function(1.0f/N),\n 1.0f,\n thrust::multiplies<float>());\n\n math::Set<float, HIPContext>(1, perplexity, Ydata, &context_);\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Perplexity, PerplexityOp<float, HIPContext>);\n} // namespace caffe2\n###"677 },678 {679 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/prepend_dim_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(PrependDim, PrependDimOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(MergeDim, MergeDimOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",680 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/prepend_dim_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(PrependDim, PrependDimOp<HIPContext>);\nREGISTER_HIP_OPERATOR(MergeDim, MergeDimOp<HIPContext>);\n\n} // namespace caffe2\n###"681 },682 {683 "cuda": "\n#include \"caffe2/operators/reciprocal_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nReciprocalGradientCUDAKernel(const int N, const T* dY, const T* Y, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * (-__ldg(Y + i) * __ldg(Y + i));\n#else\n dX[i] = dY[i] * (-Y[i] * Y[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ReciprocalGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n ReciprocalGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Reciprocal,\n UnaryElementwiseOp<\n TensorTypes<float, double>,\n CUDAContext,\n ReciprocalFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n ReciprocalGradient,\n BinaryElementwiseOp<\n TensorTypes<float, double>,\n CUDAContext,\n ReciprocalGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",684 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/reciprocal_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nReciprocalGradientHIPKernel(const int N, const T* dY, const T* Y, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * (-__ldg(Y + i) * __ldg(Y + i));\n#else\n dX[i] = dY[i] * (-Y[i] * Y[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ReciprocalGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( ReciprocalGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Reciprocal,\n UnaryElementwiseOp<\n TensorTypes<float, double>,\n HIPContext,\n ReciprocalFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n ReciprocalGradient,\n BinaryElementwiseOp<\n TensorTypes<float, double>,\n HIPContext,\n ReciprocalGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"685 },686 {687 "cuda": "\n#include \"caffe2/operators/relu_n_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nReluNCUDAKernel(const int N, const T threshold, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n Y[i] = __ldg(X + i) > 0\n ? (__ldg(X + i) < threshold ? __ldg(X + i) : threshold)\n : T(0);\n#else\n Y[i] = X[i] > 0 ? (X[i] < threshold ? X[i] : threshold) : T(0);\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void ReluNGradientCUDAKernel(\n const int N,\n const T threshold,\n const T* dY,\n const T* Y,\n T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = (__ldg(Y + i) > 0 && __ldg(Y + i) < threshold) ? dY[i] : T(0);\n#else\n dX[i] = (Y[i] > 0 && Y[i] < threshold) ? dY[i] : T(0);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ReluNFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n ReluNCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, n, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool ReluNGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n ReluNGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, n, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n ReluN,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n ReluNFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n ReluNGradient,\n BinaryElementwiseWithArgsOp<\n TensorTypes<float>,\n CUDAContext,\n ReluNGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",688 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/relu_n_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nReluNHIPKernel(const int N, const T threshold, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n Y[i] = __ldg(X + i) > 0\n ? (__ldg(X + i) < threshold ? __ldg(X + i) : threshold)\n : T(0);\n#else\n Y[i] = X[i] > 0 ? (X[i] < threshold ? X[i] : threshold) : T(0);\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void ReluNGradientHIPKernel(\n const int N,\n const T threshold,\n const T* dY,\n const T* Y,\n T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = (__ldg(Y + i) > 0 && __ldg(Y + i) < threshold) ? dY[i] : T(0);\n#else\n dX[i] = (Y[i] > 0 && Y[i] < threshold) ? dY[i] : T(0);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool ReluNFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( ReluNHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), N, n, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool ReluNGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( ReluNGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, n, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n ReluN,\n UnaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n ReluNFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n ReluNGradient,\n BinaryElementwiseWithArgsOp<\n TensorTypes<float>,\n HIPContext,\n ReluNGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"689 },690 {691 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/replace_nan_op.h\"\n\nnamespace caffe2 {\n\nnamespace {\ntemplate <typename T>\n__global__ void\nreplace_nan_kernel(const T value, const int64_t size, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, size) {\n if (isnan(X[i])) {\n Y[i] = value;\n } else {\n Y[i] = X[i];\n }\n }\n}\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nvoid ReplaceNaNOp<CUDAContext>::ReplaceNaN(\n const T& value,\n const int64_t size,\n const T* X,\n T* Y) {\n replace_nan_kernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(value, size, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\nREGISTER_CUDA_OPERATOR(ReplaceNaN, ReplaceNaNOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",692 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/replace_nan_op.h\"\n\nnamespace caffe2 {\n\nnamespace {\ntemplate <typename T>\n__global__ void\nreplace_nan_kernel(const T value, const int64_t size, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, size) {\n if (isnan(X[i])) {\n Y[i] = value;\n } else {\n Y[i] = X[i];\n }\n }\n}\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nvoid ReplaceNaNOp<HIPContext>::ReplaceNaN(\n const T& value,\n const int64_t size,\n const T* X,\n T* Y) {\n hipLaunchKernelGGL(( replace_nan_kernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), value, size, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\nREGISTER_HIP_OPERATOR(ReplaceNaN, ReplaceNaNOp<HIPContext>);\n} // namespace caffe2\n###"693 },694 {695 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/reshape_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Reshape, ReshapeOp<float, CUDAContext>);\n\n} // namespace caffe2\n\n\n###",696 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/reshape_op.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Reshape, ReshapeOp<float, HIPContext>);\n\n} // namespace caffe2\n###"697 },698 {699 "cuda": "\n#include <iostream>\n\n#include <gtest/gtest.h>\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/flags.h\"\n#include \"caffe2/operators/reshape_op.h\"\n#include \"caffe2/utils/math.h\"\n\nC10_DECLARE_string(caffe_test_root);\n\nnamespace caffe2 {\n\nstatic void AddConstInput(\n const vector<int64_t>& shape,\n const float value,\n const string& name,\n Workspace* ws) {\n DeviceOption option;\n option.set_device_type(PROTO_CUDA);\n CUDAContext context(option);\n Blob* blob = ws->CreateBlob(name);\n auto* tensor = BlobGetMutableTensor(blob, CUDA);\n tensor->Resize(shape);\n math::Set<float, CUDAContext>(\n tensor->numel(), value, tensor->template mutable_data<float>(), &context);\n return;\n}\n\nTEST(ReshapeOpGPUTest, testReshapeWithScalar) {\n if (!HasCudaGPU())\n return;\n Workspace ws;\n OperatorDef def;\n def.set_name(\"test_reshape\");\n def.set_type(\"Reshape\");\n def.add_input(\"X\");\n def.add_output(\"XNew\");\n def.add_output(\"OldShape\");\n def.add_arg()->CopyFrom(MakeArgument(\"shape\", vector<int64_t>{1}));\n def.mutable_device_option()->set_device_type(PROTO_CUDA);\n AddConstInput(vector<int64_t>(), 3.14, \"X\", &ws);\n // execute the op\n unique_ptr<OperatorBase> op(CreateOperator(def, &ws));\n EXPECT_TRUE(op->Run());\n Blob* XNew = ws.GetBlob(\"XNew\");\n const Tensor& XNewTensor = XNew->Get<Tensor>();\n EXPECT_EQ(1, XNewTensor.dim());\n EXPECT_EQ(1, XNewTensor.numel());\n}\n\n} // namespace caffe2\n\n\n###",700 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <iostream>\n\n#include <gtest/gtest.h>\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/flags.h\"\n#include \"caffe2/operators/reshape_op.h\"\n#include \"caffe2/utils/math.h\"\n\nC10_DECLARE_string(caffe_test_root);\n\nnamespace caffe2 {\n\nstatic void AddConstInput(\n const vector<int64_t>& shape,\n const float value,\n const string& name,\n Workspace* ws) {\n DeviceOption option;\n option.set_device_type(PROTO_HIP);\n HIPContext context(option);\n Blob* blob = ws->CreateBlob(name);\n auto* tensor = BlobGetMutableTensor(blob, HIP);\n tensor->Resize(shape);\n math::Set<float, HIPContext>(\n tensor->numel(), value, tensor->template mutable_data<float>(), &context);\n return;\n}\n\nTEST(ReshapeOpGPUTest, testReshapeWithScalar) {\n if (!HasHipGPU())\n return;\n Workspace ws;\n OperatorDef def;\n def.set_name(\"test_reshape\");\n def.set_type(\"Reshape\");\n def.add_input(\"X\");\n def.add_output(\"XNew\");\n def.add_output(\"OldShape\");\n def.add_arg()->CopyFrom(MakeArgument(\"shape\", vector<int64_t>{1}));\n def.mutable_device_option()->set_device_type(PROTO_HIP);\n AddConstInput(vector<int64_t>(), 3.14, \"X\", &ws);\n // execute the op\n unique_ptr<OperatorBase> op(CreateOperator(def, &ws));\n EXPECT_TRUE(op->Run());\n Blob* XNew = ws.GetBlob(\"XNew\");\n const Tensor& XNewTensor = XNew->Get<Tensor>();\n EXPECT_EQ(1, XNewTensor.dim());\n EXPECT_EQ(1, XNewTensor.numel());\n}\n\n} // namespace caffe2\n###"701 },702 {703 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/reverse_packed_segs_op.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T, typename LengthType>\n__global__\nvoid ReversePackedSegments_kernel(\n size_t max_length,\n size_t batch_size,\n size_t block_size,\n const LengthType* lengths_ptr,\n const T* data_ptr,\n T* rev_data_ptr) {\n\n const int block_id = blockIdx.x;\n\n // index into [0, batch_size)\n const int batch = block_id / max_length;\n // index into [0, segment)\n const int segment = block_id % max_length;\n\n if (batch >= batch_size || segment >= max_length) return;\n\n const int seg_length = lengths_ptr[batch];\n\n // unique data pointer for this CTA\n const T* local_data_ptr = data_ptr + (segment * batch_size + batch) * block_size;\n\n // unique pointer for result\n T* local_rev_data_ptr;\n if (segment < seg_length) {\n local_rev_data_ptr = rev_data_ptr + ((seg_length - 1 - segment) * batch_size + batch) * block_size;\n } else {\n local_rev_data_ptr = rev_data_ptr + (segment * batch_size + batch) * block_size;\n }\n\n // copy using 1 element / thread for now\n for (int idx = threadIdx.x; idx < block_size; idx+=blockDim.x) {\n local_rev_data_ptr[idx] = local_data_ptr[idx];\n }\n}\n\n} // namespace\n\n// specialization of DoRunWithLengthType\ntemplate <>\ntemplate <typename T, typename LengthType>\nvoid ReversePackedSegsOp<CUDAContext>::DoRunWithLengthType() {\n const auto& data = Input(DATA);\n const auto& lengths = Input(LENGTHS);\n\n CAFFE_ENFORCE(\n data.dim() == 3,\n \"DATA should be 3-D tensor <lengths, \"\n \"segments, embeddings>\");\n CAFFE_ENFORCE(lengths.dim() == 1, \"LENGTH should be 1-D\");\n\n auto* output = Output(0, data.sizes(), at::dtype<T>());\n\n const auto max_length = data.size(0);\n const auto batch_size = data.size(1);\n const auto block_size = data.size(2);\n CAFFE_ENFORCE(\n lengths.sizes()[0] == batch_size,\n \"lenths size should be\"\n \" equal to batch size\");\n\n const T* data_ptr = data.template data<T>();\n const LengthType* lengths_ptr = lengths.template data<LengthType>();\n\n // reversed data\n T* rev_data_ptr = output->template mutable_data<T>();\n\n const int grid = max_length * batch_size;\n\n ReversePackedSegments_kernel<T,LengthType><<<grid, 512, 0, context_.cuda_stream()>>>(\n max_length,\n batch_size,\n block_size,\n lengths_ptr,\n data_ptr,\n rev_data_ptr);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_CUDA_OPERATOR(ReversePackedSegs, ReversePackedSegsOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",704 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/reverse_packed_segs_op.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T, typename LengthType>\n__global__\nvoid ReversePackedSegments_kernel(\n size_t max_length,\n size_t batch_size,\n size_t block_size,\n const LengthType* lengths_ptr,\n const T* data_ptr,\n T* rev_data_ptr) {\n\n const int block_id = blockIdx.x;\n\n // index into [0, batch_size)\n const int batch = block_id / max_length;\n // index into [0, segment)\n const int segment = block_id % max_length;\n\n if (batch >= batch_size || segment >= max_length) return;\n\n const int seg_length = lengths_ptr[batch];\n\n // unique data pointer for this CTA\n const T* local_data_ptr = data_ptr + (segment * batch_size + batch) * block_size;\n\n // unique pointer for result\n T* local_rev_data_ptr;\n if (segment < seg_length) {\n local_rev_data_ptr = rev_data_ptr + ((seg_length - 1 - segment) * batch_size + batch) * block_size;\n } else {\n local_rev_data_ptr = rev_data_ptr + (segment * batch_size + batch) * block_size;\n }\n\n // copy using 1 element / thread for now\n for (int idx = threadIdx.x; idx < block_size; idx+=blockDim.x) {\n local_rev_data_ptr[idx] = local_data_ptr[idx];\n }\n}\n\n} // namespace\n\n// specialization of DoRunWithLengthType\ntemplate <>\ntemplate <typename T, typename LengthType>\nvoid ReversePackedSegsOp<HIPContext>::DoRunWithLengthType() {\n const auto& data = Input(DATA);\n const auto& lengths = Input(LENGTHS);\n\n CAFFE_ENFORCE(\n data.dim() == 3,\n \"DATA should be 3-D tensor <lengths, \"\n \"segments, embeddings>\");\n CAFFE_ENFORCE(lengths.dim() == 1, \"LENGTH should be 1-D\");\n\n auto* output = Output(0, data.sizes(), at::dtype<T>());\n\n const auto max_length = data.size(0);\n const auto batch_size = data.size(1);\n const auto block_size = data.size(2);\n CAFFE_ENFORCE(\n lengths.sizes()[0] == batch_size,\n \"lenths size should be\"\n \" equal to batch size\");\n\n const T* data_ptr = data.template data<T>();\n const LengthType* lengths_ptr = lengths.template data<LengthType>();\n\n // reversed data\n T* rev_data_ptr = output->template mutable_data<T>();\n\n const int grid = max_length * batch_size;\n\n hipLaunchKernelGGL(( ReversePackedSegments_kernel<T,LengthType>), dim3(grid), dim3(512), 0, context_.hip_stream(), \n max_length,\n batch_size,\n block_size,\n lengths_ptr,\n data_ptr,\n rev_data_ptr);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_HIP_OPERATOR(ReversePackedSegs, ReversePackedSegsOp<HIPContext>);\n} // namespace caffe2\n###"705 },706 {707 "cuda": "\n#include \"caffe2/operators/rsqrt_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nRsqrtGradientCUDAKernel(const int size, const T* dY, const T* Y, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, size) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * math::utils::Cube<T>(__ldg(Y + i)) *\n static_cast<T>(-0.5);\n#else\n dX[i] = dY[i] * math::utils::Cube<T>(Y[i]) * static_cast<T>(-0.5);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool RsqrtGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& dY_dims,\n const std::vector<int>& /* Y_dims */,\n const T* dY,\n const T* Y,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n dY_dims.cbegin(), dY_dims.cend(), 1, std::multiplies<int>());\n RsqrtGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Rsqrt,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n RsqrtFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n RsqrtGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n RsqrtGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",708 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/rsqrt_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/utils/math.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nRsqrtGradientHIPKernel(const int size, const T* dY, const T* Y, T* dX) {\n HIP_1D_KERNEL_LOOP(i, size) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * math::utils::Cube<T>(__ldg(Y + i)) *\n static_cast<T>(-0.5);\n#else\n dX[i] = dY[i] * math::utils::Cube<T>(Y[i]) * static_cast<T>(-0.5);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool RsqrtGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& dY_dims,\n const std::vector<int>& /* Y_dims */,\n const T* dY,\n const T* Y,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n dY_dims.cbegin(), dY_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( RsqrtGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Rsqrt,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n RsqrtFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n RsqrtGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n RsqrtGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"709 },710 {711 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid silu_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"silu_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t x_acc = static_cast<opmath_t>(x);\n return x_acc / (opmath_t(1) + c10::cuda::compat::exp(-x_acc));\n });\n });\n}\n\nvoid silu_backward_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"silu_backward_cuda\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t dy, scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t dy_acc = static_cast<opmath_t>(dy);\n const opmath_t x_acc = static_cast<opmath_t>(x);\n const opmath_t s_acc =\n opmath_t(1) / (opmath_t(1) + c10::cuda::compat::exp(-x_acc));\n return dy_acc * s_acc * (opmath_t(1) + x_acc * (opmath_t(1) - s_acc));\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(silu_stub, &silu_kernel);\nREGISTER_DISPATCH(silu_backward_stub, &silu_backward_kernel);\n\n} // namespace at::native\n\n\n###",712 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid silu_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"silu_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t x_acc = static_cast<opmath_t>(x);\n return x_acc / (opmath_t(1) + c10::hip::compat::exp(-x_acc));\n });\n });\n}\n\nvoid silu_backward_kernel(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"silu_backward_hip\",\n [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA(scalar_t dy, scalar_t x) -> scalar_t {\n using opmath_t = at::opmath_type<scalar_t>;\n const opmath_t dy_acc = static_cast<opmath_t>(dy);\n const opmath_t x_acc = static_cast<opmath_t>(x);\n const opmath_t s_acc =\n opmath_t(1) / (opmath_t(1) + c10::hip::compat::exp(-x_acc));\n return dy_acc * s_acc * (opmath_t(1) + x_acc * (opmath_t(1) - s_acc));\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(silu_stub, &silu_kernel);\nREGISTER_DISPATCH(silu_backward_stub, &silu_backward_kernel);\n\n} // namespace at::native\n###"713 },714 {715 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/scale_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool ScaleOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<at::Half, float>>::call(this, Input(0));\n}\n\nREGISTER_CUDA_OPERATOR(Scale, ScaleOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",716 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/scale_op.h\"\n\nnamespace caffe2 {\n\ntemplate <>\nbool ScaleOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<at::Half, float>>::call(this, Input(0));\n}\n\nREGISTER_HIP_OPERATOR(Scale, ScaleOp<HIPContext>);\n\n} // namespace caffe2\n###"717 },718 {719 "cuda": "\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/selu_op.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void SeluKernel(const int N, const T* X, T* Y, T alpha_, T lambda_) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = lambda_ * (X[i] > 0 ? X[i] : alpha_ * __expf(X[i]) - alpha_);\n }\n}\n\ntemplate <typename T>\n__global__ void SeluGradientKernel(\n const int N,\n const T* Y,\n const T* dY,\n T* dX,\n T alpha_,\n T lambda_) {\n const T c = lambda_ * alpha_;\n CUDA_1D_KERNEL_LOOP(i, N) {\n // Reuse Y[i] to avoid computing exp(X[i])\n dX[i] = Y[i] > 0 ? lambda_ * dY[i] : dY[i] * (Y[i] + c);\n }\n}\n} // namespace\n\ntemplate <>\nbool SeluOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n SeluKernel<float>\n <<<CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(),\n X.data<float>(),\n Y->template mutable_data<float>(),\n alpha_,\n lambda_);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool SeluGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n CAFFE_ENFORCE_GT(Y.numel(), 0);\n CAFFE_ENFORCE_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n SeluGradientKernel<float>\n <<<CAFFE_GET_BLOCKS(Y.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n Y.numel(),\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>(),\n alpha_,\n lambda_);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Selu, SeluOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(SeluGradient, SeluGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",720 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/selu_op.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void SeluKernel(const int N, const T* X, T* Y, T alpha_, T lambda_) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = lambda_ * (X[i] > 0 ? X[i] : alpha_ * expf(X[i]) - alpha_);\n }\n}\n\ntemplate <typename T>\n__global__ void SeluGradientKernel(\n const int N,\n const T* Y,\n const T* dY,\n T* dX,\n T alpha_,\n T lambda_) {\n const T c = lambda_ * alpha_;\n HIP_1D_KERNEL_LOOP(i, N) {\n // Reuse Y[i] to avoid computing exp(X[i])\n dX[i] = Y[i] > 0 ? lambda_ * dY[i] : dY[i] * (Y[i] + c);\n }\n}\n} // namespace\n\ntemplate <>\nbool SeluOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( SeluKernel<float>)\n , dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(),\n X.data<float>(),\n Y->template mutable_data<float>(),\n alpha_,\n lambda_);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool SeluGradientOp<float, HIPContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n CAFFE_ENFORCE_GT(Y.numel(), 0);\n CAFFE_ENFORCE_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( SeluGradientKernel<float>)\n , dim3(CAFFE_GET_BLOCKS(Y.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n Y.numel(),\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>(),\n alpha_,\n lambda_);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Selu, SeluOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(SeluGradient, SeluGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"721 },722 {723 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/shape_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(Shape, ShapeOp<CUDAContext>);\n}\n\n\n###",724 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/shape_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(Shape, ShapeOp<HIPContext>);\n}\n###"725 },726 {727 "cuda": "\n#include \"caffe2/operators/sigmoid_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void SigmoidCUDAKernel(const int N, const T* X, T* Y);\n\ntemplate <>\n__global__ void\nSigmoidCUDAKernel<float>(const int N, const float* X, float* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n Y[i] = 1.0f / (1.0f + expf(-__ldg(X + i)));\n#else\n Y[i] = 1.0f / (1.0f + expf(-X[i]));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void\nSigmoidGradientCUDAKernel(const int N, const T* dY, const T* Y, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * __ldg(Y + i) * (T(1) - __ldg(Y + i));\n#else\n dX[i] = dY[i] * Y[i] * (T(1) - Y[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SigmoidFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n SigmoidCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool SigmoidGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n SigmoidGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Sigmoid,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SigmoidFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n SigmoidGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SigmoidGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",728 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/sigmoid_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void SigmoidHIPKernel(const int N, const T* X, T* Y);\n\ntemplate <>\n__global__ void\nSigmoidHIPKernel<float>(const int N, const float* X, float* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n Y[i] = 1.0f / (1.0f + expf(-__ldg(X + i)));\n#else\n Y[i] = 1.0f / (1.0f + expf(-X[i]));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void\nSigmoidGradientHIPKernel(const int N, const T* dY, const T* Y, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * __ldg(Y + i) * (T(1) - __ldg(Y + i));\n#else\n dX[i] = dY[i] * Y[i] * (T(1) - Y[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SigmoidFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( SigmoidHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), N, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool SigmoidGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( SigmoidGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Sigmoid,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SigmoidFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n SigmoidGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SigmoidGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"729 },730 {731 "cuda": "\n#include \"caffe2/operators/sinh_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void SinhGradientCUDAKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * coshf(__ldg(X + i));\n#else\n dX[i] = dY[i] * coshf(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SinhGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& /* dY_dims */,\n const std::vector<int>& X_dims,\n const T* dY,\n const T* X,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n SinhGradientCUDAKernel<<<\n CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Sinh,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SinhFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n SinhGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SinhGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",732 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/sinh_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\n__global__ void SinhGradientHIPKernel(\n const int N,\n const float* dY,\n const float* X,\n float* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * coshf(__ldg(X + i));\n#else\n dX[i] = dY[i] * coshf(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SinhGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& /* dY_dims */,\n const std::vector<int>& X_dims,\n const T* dY,\n const T* X,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( SinhGradientHIPKernel), \n dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Sinh,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SinhFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n SinhGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SinhGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"733 },734 {735 "cuda": "\n#include \"caffe2/operators/sin_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nSinGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * cos(__ldg(X + i));\n#else\n dX[i] = dY[i] * cos(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SinGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n SinGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Sin,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SinFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n SinGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SinGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",736 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/sin_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nSinGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * cos(__ldg(X + i));\n#else\n dX[i] = dY[i] * cos(X[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SinGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( SinGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Sin,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SinFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n SinGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SinGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"737 },738 {739 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/softplus_op.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void SoftplusKernel(const int N, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = log(exp(X[i]) + 1.0f);\n }\n}\n\ntemplate <typename T>\n__global__ void\nSoftplusGradientKernel(const int N, const T* Y, const T* dY, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n const float nexpY = exp(-Y[i]);\n dX[i] = dY[i] * (1 - nexpY);\n }\n}\n} // namespace\n\ntemplate <>\nbool SoftplusOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n TORCH_DCHECK_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n SoftplusKernel<float>\n <<<CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool SoftplusGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n TORCH_DCHECK_GT(Y.numel(), 0);\n TORCH_DCHECK_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n SoftplusGradientKernel<float>\n <<<CAFFE_GET_BLOCKS(Y.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n Y.numel(),\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(Softplus, SoftplusOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n SoftplusGradient,\n SoftplusGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",740 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/softplus_op.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void SoftplusKernel(const int N, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = log(exp(X[i]) + 1.0f);\n }\n}\n\ntemplate <typename T>\n__global__ void\nSoftplusGradientKernel(const int N, const T* Y, const T* dY, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n const float nexpY = exp(-Y[i]);\n dX[i] = dY[i] * (1 - nexpY);\n }\n}\n} // namespace\n\ntemplate <>\nbool SoftplusOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n TORCH_DCHECK_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( SoftplusKernel<float>)\n , dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(), X.data<float>(), Y->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool SoftplusGradientOp<float, HIPContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n TORCH_DCHECK_GT(Y.numel(), 0);\n TORCH_DCHECK_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( SoftplusGradientKernel<float>)\n , dim3(CAFFE_GET_BLOCKS(Y.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n Y.numel(),\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(Softplus, SoftplusOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n SoftplusGradient,\n SoftplusGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"741 },742 {743 "cuda": "\n#include \"caffe2/operators/softsign_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\nusing c10::cuda::compat::abs;\n\ntemplate <typename T>\ninline __host__ __device__ T SquareCUDA(const T x) {\n return x * x;\n}\n\ntemplate <typename T>\n__global__ void SoftsignCUDAKernel(const int N, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n Y[i] = __ldg(X + i) / (T(1) + abs(__ldg(X + i)));\n#else\n Y[i] = X[i] / (T(1) + abs(X[i]));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void\nSoftsignGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / SquareCUDA(T(1) + abs(__ldg(X + i)));\n#else\n dX[i] = dY[i] / SquareCUDA(T(1) + abs(X[i]));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SoftsignFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n SoftsignCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool SoftsignGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n SoftsignGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Softsign,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SoftsignFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n SoftsignGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SoftsignGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",744 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/softsign_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\nusing c10::hip::compat::abs;\n\ntemplate <typename T>\ninline __host__ __device__ T SquareHIP(const T x) {\n return x * x;\n}\n\ntemplate <typename T>\n__global__ void SoftsignHIPKernel(const int N, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n Y[i] = __ldg(X + i) / (T(1) + abs(__ldg(X + i)));\n#else\n Y[i] = X[i] / (T(1) + abs(X[i]));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void\nSoftsignGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / SquareHIP(T(1) + abs(__ldg(X + i)));\n#else\n dX[i] = dY[i] / SquareHIP(T(1) + abs(X[i]));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SoftsignFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( SoftsignHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), N, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool SoftsignGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( SoftsignGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Softsign,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SoftsignFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n SoftsignGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SoftsignGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"745 },746 {747 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/operator_fallback_gpu.h\"\n#include \"caffe2/operators/sparse_lp_regularizer_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(SparseLpRegularizer, GPUFallbackOp);\n}\n\n\n###",748 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/hip/operator_fallback_gpu.h\"\n#include \"caffe2/operators/sparse_lp_regularizer_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(SparseLpRegularizer, GPUFallbackOp);\n}\n###"749 },750 {751 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/operator_fallback_gpu.h\"\n#include \"caffe2/operators/sparse_normalize_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(\n SparseNormalize,\n GPUFallbackOp);\n}\n\n\n###",752 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/hip/operator_fallback_gpu.h\"\n#include \"caffe2/operators/sparse_normalize_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(\n SparseNormalize,\n GPUFallbackOp);\n}\n###"753 },754 {755 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid softplus_kernel(\n TensorIteratorBase& iter,\n const Scalar& beta_,\n const Scalar& threshold_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"softplus_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto beta = beta_.to<opmath_t>();\n auto threshold = threshold_.to<opmath_t>();\n gpu_kernel(iter, [beta, threshold] GPU_LAMBDA(scalar_t a) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n return (aop * beta) > threshold\n ? aop\n : (::log1p(std::exp(aop * beta))) / beta;\n });\n });\n}\n\nvoid softplus_backward_kernel(\n TensorIteratorBase& iter,\n const Scalar& beta_,\n const Scalar& threshold_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"softplus_backward_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto beta = beta_.to<opmath_t>();\n auto threshold = threshold_.to<opmath_t>();\n gpu_kernel(\n iter,\n [beta, threshold] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n opmath_t z = std::exp(bop * beta);\n return (bop * beta) > threshold ? aop\n : aop * z / (z + opmath_t(1.));\n });\n });\n}\n\n} // namespace\n\nREGISTER_DISPATCH(softplus_stub, &softplus_kernel);\nREGISTER_DISPATCH(softplus_backward_stub, &softplus_backward_kernel);\n\n} // namespace at::native\n\n\n###",756 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid softplus_kernel(\n TensorIteratorBase& iter,\n const Scalar& beta_,\n const Scalar& threshold_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"softplus_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto beta = beta_.to<opmath_t>();\n auto threshold = threshold_.to<opmath_t>();\n gpu_kernel(iter, [beta, threshold] GPU_LAMBDA(scalar_t a) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n return (aop * beta) > threshold\n ? aop\n : (::log1p(::exp(aop * beta))) / beta;\n });\n });\n}\n\nvoid softplus_backward_kernel(\n TensorIteratorBase& iter,\n const Scalar& beta_,\n const Scalar& threshold_) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"softplus_backward_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto beta = beta_.to<opmath_t>();\n auto threshold = threshold_.to<opmath_t>();\n gpu_kernel(\n iter,\n [beta, threshold] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n opmath_t aop = static_cast<opmath_t>(a);\n opmath_t bop = static_cast<opmath_t>(b);\n opmath_t z = ::exp(bop * beta);\n return (bop * beta) > threshold ? aop\n : aop * z / (z + opmath_t(1.));\n });\n });\n}\n\n} // namespace\n\nREGISTER_DISPATCH(softplus_stub, &softplus_kernel);\nREGISTER_DISPATCH(softplus_backward_stub, &softplus_backward_kernel);\n\n} // namespace at::native\n###"757 },758 {759 "cuda": "\n#include \"caffe2/operators/sparse_to_dense_op.h\"\n\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/utils/GpuAtomics.cuh\"\n\nnamespace caffe2 {\n\n template <typename TInd, typename TData>\n __global__ void SparseToDenseKernel(\n size_t N, int64_t block_nitems, const TInd* indices, const TData* vals, TData* dst) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n int idx = indices[i / block_nitems];\n int dst_idx = block_nitems * idx + i % block_nitems;\n gpu_atomic_add(&dst[dst_idx], vals[i]);\n }\n }\n\n template <>\n bool SparseToDenseOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<int32_t>>::call(\n this, Input(INDICES));\n }\n\n template <>\n template <typename TInd>\n bool SparseToDenseOp<CUDAContext>::DoRunWithType() {\n return DispatchHelper<\n TensorTypes2<\n float,\n int32_t>,\n TInd>::call(this, Input(VALUES));\n }\n\n template <>\n template <typename TInd, typename TData>\n bool SparseToDenseOp<CUDAContext>::DoRunWithType2() {\n auto& sparse_indices = Input(INDICES);\n CAFFE_ENFORCE_EQ(sparse_indices.dim(), 1);\n auto& sparse_values = Input(VALUES);\n CAFFE_ENFORCE_GE(sparse_values.dim(), 1);\n CAFFE_ENFORCE_EQ(sparse_indices.numel(), sparse_values.dim(0));\n\n const TInd* sparse_indices_vec = sparse_indices.template data<TInd>();\n const int32_t sparse_indices_len = sparse_indices.dim32(0);\n const int output_first_dim =\n GetOutputFirstDim(sparse_indices_vec, sparse_indices_len);\n\n auto shape = sparse_values.sizes().vec();\n shape[0] = output_first_dim;\n\n auto* output = Output(0, shape, at::dtype<TData>());\n\n TData* output_data = output->template mutable_data<TData>();\n math::Set<TData>(output->numel(), TData(0), output_data, &context_);\n\n const auto block_nitems = sparse_values.size_from_dim(1);\n const TData* sparse_values_vec = sparse_values.template data<TData>();\n\n size_t N = block_nitems * sparse_indices_len;\n CAFFE_ENFORCE_EQ(output->numel(), output_first_dim * block_nitems);\n SparseToDenseKernel<TInd, TData><<<\n CAFFE_GET_BLOCKS(N), CAFFE_CUDA_NUM_THREADS, 0,\n context_.cuda_stream()>>>(\n N,\n block_nitems,\n sparse_indices_vec,\n sparse_values_vec,\n output_data\n );\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n }\n\n\nREGISTER_CUDA_OPERATOR(SparseToDense, SparseToDenseOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",760 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/sparse_to_dense_op.h\"\n\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/utils/hip/GpuAtomics.cuh\"\n\nnamespace caffe2 {\n\n template <typename TInd, typename TData>\n __global__ void SparseToDenseKernel(\n size_t N, int64_t block_nitems, const TInd* indices, const TData* vals, TData* dst) {\n HIP_1D_KERNEL_LOOP(i, N) {\n int idx = indices[i / block_nitems];\n int dst_idx = block_nitems * idx + i % block_nitems;\n gpu_atomic_add(&dst[dst_idx], vals[i]);\n }\n }\n\n template <>\n bool SparseToDenseOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<int32_t>>::call(\n this, Input(INDICES));\n }\n\n template <>\n template <typename TInd>\n bool SparseToDenseOp<HIPContext>::DoRunWithType() {\n return DispatchHelper<\n TensorTypes2<\n float,\n int32_t>,\n TInd>::call(this, Input(VALUES));\n }\n\n template <>\n template <typename TInd, typename TData>\n bool SparseToDenseOp<HIPContext>::DoRunWithType2() {\n auto& sparse_indices = Input(INDICES);\n CAFFE_ENFORCE_EQ(sparse_indices.dim(), 1);\n auto& sparse_values = Input(VALUES);\n CAFFE_ENFORCE_GE(sparse_values.dim(), 1);\n CAFFE_ENFORCE_EQ(sparse_indices.numel(), sparse_values.dim(0));\n\n const TInd* sparse_indices_vec = sparse_indices.template data<TInd>();\n const int32_t sparse_indices_len = sparse_indices.dim32(0);\n const int output_first_dim =\n GetOutputFirstDim(sparse_indices_vec, sparse_indices_len);\n\n auto shape = sparse_values.sizes().vec();\n shape[0] = output_first_dim;\n\n auto* output = Output(0, shape, at::dtype<TData>());\n\n TData* output_data = output->template mutable_data<TData>();\n math::Set<TData>(output->numel(), TData(0), output_data, &context_);\n\n const auto block_nitems = sparse_values.size_from_dim(1);\n const TData* sparse_values_vec = sparse_values.template data<TData>();\n\n size_t N = block_nitems * sparse_indices_len;\n CAFFE_ENFORCE_EQ(output->numel(), output_first_dim * block_nitems);\n hipLaunchKernelGGL(( SparseToDenseKernel<TInd, TData>), \n dim3(CAFFE_GET_BLOCKS(N)), dim3(CAFFE_HIP_NUM_THREADS), 0,\n context_.hip_stream(), \n N,\n block_nitems,\n sparse_indices_vec,\n sparse_values_vec,\n output_data\n );\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n }\n\n\nREGISTER_HIP_OPERATOR(SparseToDense, SparseToDenseOp<HIPContext>);\n\n} // namespace caffe2\n###"761 },762 {763 "cuda": "\n#include \"caffe2/operators/spatial_batch_norm_op.h\"\n\n#include \"caffe2/operators/spatial_batch_norm_op_impl.cuh\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(SpatialBN, SpatialBNOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(SpatialBNGradient, SpatialBNGradientOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",764 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/spatial_batch_norm_op.h\"\n\n#include \"caffe2/operators/hip/spatial_batch_norm_op_impl.cuh\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(SpatialBN, SpatialBNOp<HIPContext>);\nREGISTER_HIP_OPERATOR(SpatialBNGradient, SpatialBNGradientOp<HIPContext>);\n\n} // namespace caffe2\n###"765 },766 {767 "cuda": "\n#include \"caffe2/operators/sqrt_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Sqrt,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SqrtFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",768 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/sqrt_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Sqrt,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SqrtFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"769 },770 {771 "cuda": "\n#include \"caffe2/operators/sqr_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(\n Sqr,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n SqrFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",772 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/sqr_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(\n Sqr,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n SqrFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"773 },774 {775 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/stop_gradient.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(StopGradient, StopGradientOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",776 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/stop_gradient.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(StopGradient, StopGradientOp<HIPContext>);\n} // namespace caffe2\n###"777 },778 {779 "cuda": "\n/**\n * Copyright (c) 2016-present, Facebook, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/stump_func_op.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename TIN, typename TOUT>\n__global__ void StumpFuncKernel(\n const int N,\n const TIN threshold,\n const TOUT low_value,\n const TOUT high_value,\n const TIN* X,\n TOUT* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = (X[i] <= threshold) ? low_value : high_value;\n }\n}\n\n} //\n\ntemplate <>\nbool StumpFuncOp<float, float, CUDAContext>::RunOnDevice() {\n auto& in = Input(0);\n const float* in_data = in.data<float>();\n\n auto* out = Output(0, in.sizes(), at::dtype<float>());\n float* out_data = out->template mutable_data<float>();\n StumpFuncKernel<<<CAFFE_GET_BLOCKS(in.numel()), CAFFE_CUDA_NUM_THREADS,\n 0, context_.cuda_stream()>>>(\n in.numel(), threshold_, low_value_, high_value_, in_data, out_data);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(StumpFunc, StumpFuncOp<float, float, CUDAContext>);\n// NO_GRADIENT(StumpFuncGpu);\n\n} // caffe2\n\n\n###",780 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n/**\n * Copyright (c) 2016-present, Facebook, Inc.\n *\n * Licensed under the Apache License, Version 2.0 (the \"License\");\n * you may not use this file except in compliance with the License.\n * You may obtain a copy of the License at\n *\n * http://www.apache.org/licenses/LICENSE-2.0\n *\n * Unless required by applicable law or agreed to in writing, software\n * distributed under the License is distributed on an \"AS IS\" BASIS,\n * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n * See the License for the specific language governing permissions and\n * limitations under the License.\n */\n\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/stump_func_op.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename TIN, typename TOUT>\n__global__ void StumpFuncKernel(\n const int N,\n const TIN threshold,\n const TOUT low_value,\n const TOUT high_value,\n const TIN* X,\n TOUT* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = (X[i] <= threshold) ? low_value : high_value;\n }\n}\n\n} //\n\ntemplate <>\nbool StumpFuncOp<float, float, HIPContext>::RunOnDevice() {\n auto& in = Input(0);\n const float* in_data = in.data<float>();\n\n auto* out = Output(0, in.sizes(), at::dtype<float>());\n float* out_data = out->template mutable_data<float>();\n hipLaunchKernelGGL(( StumpFuncKernel), dim3(CAFFE_GET_BLOCKS(in.numel())), dim3(CAFFE_HIP_NUM_THREADS),\n 0, context_.hip_stream(), \n in.numel(), threshold_, low_value_, high_value_, in_data, out_data);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(StumpFunc, StumpFuncOp<float, float, HIPContext>);\n// NO_GRADIENT(StumpFuncGpu);\n\n} // caffe2\n###"781 },782 {783 "cuda": "\n#include <cuda.h>\n#include <thrust/device_vector.h>\n#include <thrust/transform_reduce.h>\n#include <thrust/system/cuda/execution_policy.h>\n#include \"caffe2/operators/summarize_op.h\"\n#include \"caffe2/core/context_gpu.h\"\nnamespace caffe2 {\nnamespace {\n\n\ntemplate <typename T>\nstruct SummaryStatsData {\n T n;\n T min;\n T max;\n T mean;\n T M2;\n \n void initialize() {\n n = mean = M2 = 0;\n min = std::numeric_limits<T>::max();\n max = std::numeric_limits<T>::min();\n }\n T variance() { return (n == 1 ? 0 : M2 / (n - 1)); }\n};\n\n\ntemplate <typename T>\nstruct summary_stats_unary_op {\n __host__ __device__ SummaryStatsData<T> operator()(const T& x) const {\n SummaryStatsData<T> result;\n result.n = 1;\n result.min = x;\n result.max = x;\n result.mean = x;\n result.M2 = 0;\n return result;\n }\n};\n\n\n\n\ntemplate <typename T>\nstruct summary_stats_binary_op\n : public thrust::binary_function<const SummaryStatsData<T>&, const SummaryStatsData<T>&, SummaryStatsData<T> > {\n __host__ __device__ SummaryStatsData<T> operator()(\n const SummaryStatsData<T>& x, const SummaryStatsData <T>& y) const {\n SummaryStatsData<T> result;\n T n = x.n + y.n;\n T delta = y.mean - x.mean;\n T delta2 = delta * delta;\n result.n = n;\n result.min = thrust::min(x.min, y.min);\n result.max = thrust::max(x.max, y.max);\n result.mean = x.mean + delta * y.n / n;\n result.M2 = x.M2 + y.M2;\n result.M2 += delta2 * x.n * y.n / n;\n return result;\n }\n};\n} \ntemplate<>\nbool SummarizeOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n const int N = X.numel();\n TORCH_DCHECK_GT(N, 0);\n \n thrust::device_ptr<float> Xdata(const_cast<float*>(X.data<float>()));\n summary_stats_unary_op<float> unary_op;\n summary_stats_binary_op<float> binary_op;\n SummaryStatsData<float> init;\n init.initialize();\n \n SummaryStatsData<float> result = thrust::transform_reduce(\n#if THRUST_VERSION >= 100800\n thrust::cuda::par.on(context_.cuda_stream()), #endif \n Xdata, Xdata + N, unary_op, init, binary_op);\n float standard_deviation = std::sqrt(result.variance());\n if (to_file_) {\n (*log_file_) << result.min << \" \" << result.max << \" \" << result.mean << \" \"\n << standard_deviation << std::endl;\n }\n if (OutputSize()) {\n auto* Y = Output(0, {4}, at::dtype<float>());\n float output_buffer[NUM_STATS] = {result.min, result.max, result.mean, standard_deviation};\n context_.CopyFromCPU<float>(\n NUM_STATS, output_buffer, Y->template mutable_data<float>());\n }\n return true;\n}\nREGISTER_CUDA_OPERATOR(Summarize, SummarizeOp<float, CUDAContext>);\n} \n\n###",784 "hip": " \n#include <hip/hip_runtime.h>\n#include <thrust/device_vector.h>\n#include <thrust/transform_reduce.h>\n#include <thrust/system/hip/execution_policy.h>\n#include \"caffe2/operators/summarize_op.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\nnamespace caffe2 {\nnamespace {\n\n\ntemplate <typename T>\nstruct SummaryStatsData {\n T n;\n T min;\n T max;\n T mean;\n T M2;\n \n void initialize() {\n n = mean = M2 = 0;\n min = std::numeric_limits<T>::max();\n max = std::numeric_limits<T>::min();\n }\n T variance() { return (n == 1 ? 0 : M2 / (n - 1)); }\n};\n\n\ntemplate <typename T>\nstruct summary_stats_unary_op {\n __host__ __device__ SummaryStatsData<T> operator()(const T& x) const {\n SummaryStatsData<T> result;\n result.n = 1;\n result.min = x;\n result.max = x;\n result.mean = x;\n result.M2 = 0;\n return result;\n }\n};\n\n\n\n\ntemplate <typename T>\nstruct summary_stats_binary_op\n : public thrust::binary_function<const SummaryStatsData<T>&, const SummaryStatsData<T>&, SummaryStatsData<T> > {\n __host__ __device__ SummaryStatsData<T> operator()(\n const SummaryStatsData<T>& x, const SummaryStatsData <T>& y) const {\n SummaryStatsData<T> result;\n T n = x.n + y.n;\n T delta = y.mean - x.mean;\n T delta2 = delta * delta;\n result.n = n;\n result.min = thrust::min(x.min, y.min);\n result.max = thrust::max(x.max, y.max);\n result.mean = x.mean + delta * y.n / n;\n result.M2 = x.M2 + y.M2;\n result.M2 += delta2 * x.n * y.n / n;\n return result;\n }\n};\n} \ntemplate<>\nbool SummarizeOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n const int N = X.numel();\n TORCH_DCHECK_GT(N, 0);\n \n thrust::device_ptr<float> Xdata(const_cast<float*>(X.data<float>()));\n summary_stats_unary_op<float> unary_op;\n summary_stats_binary_op<float> binary_op;\n SummaryStatsData<float> init;\n init.initialize();\n \n SummaryStatsData<float> result = thrust::transform_reduce(\n#if THRUST_VERSION >= 100800\n thrust::hip::par.on(context_.hip_stream()), #endif \n Xdata, Xdata + N, unary_op, init, binary_op);\n float standard_deviation = std::sqrt(result.variance());\n if (to_file_) {\n (*log_file_) << result.min << \" \" << result.max << \" \" << result.mean << \" \"\n << standard_deviation << std::endl;\n }\n if (OutputSize()) {\n auto* Y = Output(0, {4}, at::dtype<float>());\n float output_buffer[NUM_STATS] = {result.min, result.max, result.mean, standard_deviation};\n context_.CopyFromCPU<float>(\n NUM_STATS, output_buffer, Y->template mutable_data<float>());\n }\n return true;\n}\nREGISTER_HIP_OPERATOR(Summarize, SummarizeOp<float, HIPContext>);\n} ###"785 },786 {787 "cuda": "\n#include \"caffe2/operators/swish_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void SwishCUDAKernel(const int N, const T* X, T* Y) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n Y[i] = __ldg(X + i) / (T(1) + exp(-__ldg(X + i)));\n#else\n Y[i] = X[i] / (T(1) + exp(-X[i]));\n#endif\n }\n}\n\ntemplate <typename T>\n__global__ void SwishGradientCUDAKernel(\n const int N,\n const T* X,\n const T* Y,\n const T* dY,\n T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) *\n (__ldg(Y + i) + (T(1) - __ldg(Y + i)) / (T(1) + exp(-__ldg(X + i))));\n#else\n dX[i] = dY[i] * (Y[i] + (T(1) - Y[i]) / (T(1) + exp(-X[i])));\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool SwishFunctor<CUDAContext>::\noperator()(const int N, const T* X, T* Y, CUDAContext* context) const {\n SwishCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(N),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(N, X, Y);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\ntemplate <typename T>\nbool SwishGradientOp<CUDAContext>::DoRunWithType() {\n auto& Xin = Input(X);\n auto& Yin = Input(Y);\n auto& DYin = Input(DY);\n auto* DXout = Output(DX);\n CAFFE_ENFORCE_EQ(Xin.size(), Yin.size());\n CAFFE_ENFORCE_EQ(DYin.size(), Yin.size());\n DXout->ResizeLike(Yin);\n\n const int n = Xin.size();\n const T* x = Xin.template data<T>();\n const T* y = Yin.template data<T>();\n const T* dy = DYin.template data<T>();\n T* dx = DXout->template mutable_data<T>();\n SwishGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(n),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(n, x, y, dy, dx);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool SwishGradientOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, double>>::call(this, Input(X));\n}\n\nREGISTER_CUDA_OPERATOR(\n Swish,\n UnaryElementwiseOp<\n TensorTypes<float, double>,\n CUDAContext,\n SwishFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(SwishGradient, SwishGradientOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",788 "hip": " \n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/swish_op.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void SwishHIPKernel(const int N, const T* X, T* Y) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n Y[i] = __ldg(X + i) / (T(1) + exp(-__ldg(X + i)));\n#else\n Y[i] = X[i] / (T(1) + exp(-X[i]));\n#endif\n }\n}\ntemplate <typename T>\n__global__ void SwishGradientHIPKernel(\n const int N, const T* X, const T* Y, const T* dY, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) *\n (__ldg(Y + i) + (T(1) - __ldg(Y + i)) / (T(1) + exp(-__ldg(X + i))));\n#else\n dX[i] = dY[i] * (Y[i] + (T(1) - Y[i]) / (T(1) + exp(-X[i])));\n#endif\n }\n}\n} \ntemplate <>\ntemplate <typename T>\nbool SwishFunctor<HIPContext>::\noperator()(const int N, const T* X, T* Y, HIPContext* context) const {\n hipLaunchKernelGGL(( SwishHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(N)), dim3(CAFFE_HIP_NUM_THREADS), 0, context->hip_stream(), N, X, Y);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n return true;\n}\ntemplate <>\ntemplate <typename T>\nbool SwishGradientOp<HIPContext>::DoRunWithType() {\n auto& Xin = Input(X);\n auto& Yin = Input(Y);\n auto& DYin = Input(DY);\n auto* DXout = Output(DX);\n CAFFE_ENFORCE_EQ(Xin.size(), Yin.size());\n CAFFE_ENFORCE_EQ(DYin.size(), Yin.size());\n DXout->ResizeLike(Yin);\n const int n = Xin.size();\n const T* x = Xin.template data<T>();\n const T* y = Yin.template data<T>();\n const T* dy = DYin.template data<T>();\n T* dx = DXout->template mutable_data<T>();\n hipLaunchKernelGGL(( SwishGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(n)), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), n, x, y, dy, dx);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n return true;\n}\ntemplate <>\nbool SwishGradientOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, double>>::call(this, Input(X));\n}\nREGISTER_HIP_OPERATOR(\n Swish, UnaryElementwiseOp<\n TensorTypes<float, double>, HIPContext, SwishFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(SwishGradient, SwishGradientOp<HIPContext>);\n} ###"789 },790 {791 "cuda": "\n#include \"caffe2/operators/tanh_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nTanhGradientCUDAKernel(const int N, const T* dY, const T* Y, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * (T(1) - __ldg(Y + i) * __ldg(Y + i));\n#else\n dX[i] = dY[i] * (T(1) - Y[i] * Y[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool TanhGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n TanhGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, Y, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Tanh,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n TanhFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n TanhGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n TanhGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",792 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/tanh_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\n__global__ void\nTanhGradientHIPKernel(const int N, const T* dY, const T* Y, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) * (T(1) - __ldg(Y + i) * __ldg(Y + i));\n#else\n dX[i] = dY[i] * (T(1) - Y[i] * Y[i]);\n#endif\n }\n}\n\n} // namespace\n\ntemplate <>\ntemplate <typename T>\nbool TanhGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& Y_dims,\n const std::vector<int>& /* dY_dims */,\n const T* Y,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n Y_dims.cbegin(), Y_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( TanhGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, Y, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Tanh,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n TanhFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n TanhGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n TanhGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"793 },794 {795 "cuda": "\n#include \"caffe2/operators/tan_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\ninline __host__ __device__ T Square(const T& x) {\n return x * x;\n}\n\ntemplate <typename T>\n__global__ void\nTanGradientCUDAKernel(const int N, const T* dY, const T* X, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n#if __CUDA_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / Square(cos(__ldg(X + i)));\n#else\n dX[i] = dY[i] / Square(cos(X[i]));\n#endif\n }\n}\n\ntemplate <>\ntemplate <typename T>\nbool TanGradientFunctor<CUDAContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n CUDAContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n TanGradientCUDAKernel<T>\n <<<CAFFE_GET_BLOCKS(size),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context->cuda_stream()>>>(size, dY, X, dX);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(\n Tan,\n UnaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n TanFunctor<CUDAContext>>);\nREGISTER_CUDA_OPERATOR(\n TanGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n CUDAContext,\n TanGradientFunctor<CUDAContext>>);\n\n} // namespace caffe2\n\n\n###",796 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/operators/tan_op.h\"\n\n#include <algorithm>\n#include <functional>\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\ntemplate <typename T>\ninline __host__ __device__ T Square(const T& x) {\n return x * x;\n}\n\ntemplate <typename T>\n__global__ void\nTanGradientHIPKernel(const int N, const T* dY, const T* X, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n#if __HIP_ARCH__ >= 350\n dX[i] = __ldg(dY + i) / Square(cos(__ldg(X + i)));\n#else\n dX[i] = dY[i] / Square(cos(X[i]));\n#endif\n }\n}\n\ntemplate <>\ntemplate <typename T>\nbool TanGradientFunctor<HIPContext>::Forward(\n const std::vector<int>& X_dims,\n const std::vector<int>& /* dY_dims */,\n const T* X,\n const T* dY,\n T* dX,\n HIPContext* context) const {\n const int size = std::accumulate(\n X_dims.cbegin(), X_dims.cend(), 1, std::multiplies<int>());\n hipLaunchKernelGGL(( TanGradientHIPKernel<T>)\n , dim3(CAFFE_GET_BLOCKS(size)),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context->hip_stream(), size, dY, X, dX);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(\n Tan,\n UnaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n TanFunctor<HIPContext>>);\nREGISTER_HIP_OPERATOR(\n TanGradient,\n BinaryElementwiseOp<\n TensorTypes<float>,\n HIPContext,\n TanGradientFunctor<HIPContext>>);\n\n} // namespace caffe2\n###"797 },798 {799 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid softshrink_kernel(TensorIteratorBase& iter, const Scalar& value) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"softshrink_cuda\",\n [&]() {\n auto lambd = value.to<scalar_t>();\n gpu_kernel(iter, [lambd] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return a > lambd ? a - lambd : (a < -lambd ? a + lambd : scalar_t(0));\n });\n });\n}\n\nvoid shrink_backward_kernel(TensorIteratorBase& iter, const Scalar& value) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"shrink_backward_cuda\",\n [&]() {\n auto lambd = value.to<scalar_t>();\n gpu_kernel(\n iter,\n [lambd] GPU_LAMBDA(\n scalar_t grad_val, scalar_t self_val) -> scalar_t {\n return (self_val >= -lambd && self_val <= lambd) ? scalar_t(0)\n : grad_val;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(softshrink_stub, &softshrink_kernel);\nREGISTER_DISPATCH(shrink_backward_stub, &shrink_backward_kernel);\n\n} // namespace at::native\n\n\n###",800 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\nvoid softshrink_kernel(TensorIteratorBase& iter, const Scalar& value) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"softshrink_hip\",\n [&]() {\n auto lambd = value.to<scalar_t>();\n gpu_kernel(iter, [lambd] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return a > lambd ? a - lambd : (a < -lambd ? a + lambd : scalar_t(0));\n });\n });\n}\n\nvoid shrink_backward_kernel(TensorIteratorBase& iter, const Scalar& value) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"shrink_backward_hip\",\n [&]() {\n auto lambd = value.to<scalar_t>();\n gpu_kernel(\n iter,\n [lambd] GPU_LAMBDA(\n scalar_t grad_val, scalar_t self_val) -> scalar_t {\n return (self_val >= -lambd && self_val <= lambd) ? scalar_t(0)\n : grad_val;\n });\n });\n}\n} // namespace\n\nREGISTER_DISPATCH(softshrink_stub, &softshrink_kernel);\nREGISTER_DISPATCH(shrink_backward_stub, &shrink_backward_kernel);\n\n} // namespace at::native\n###"801 },802 {803 "cuda": "\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/tensor_protos_db_input.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(TensorProtosDBInput, TensorProtosDBInput<CUDAContext>);\n} // namespace caffe2\n\n\n###",804 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/tensor_protos_db_input.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(TensorProtosDBInput, TensorProtosDBInput<HIPContext>);\n} // namespace caffe2\n###"805 },806 {807 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/thresholded_relu_op.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void ThresholdedReluKernel(const int N, const T* X, T* Y, T alpha_) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n Y[i] = X[i] > alpha_ ? X[i] : 0;\n }\n}\n\ntemplate <typename T>\n__global__ void\nThresholdedReluGradientKernel(const int N, const T* Y, const T* dY, T* dX) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n dX[i] = Y[i] > 0 ? dY[i] : 0;\n }\n}\n} // namespace\n\ntemplate <>\nbool ThresholdedReluOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n ThresholdedReluKernel<<<\n CAFFE_GET_BLOCKS(X.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n X.numel(), X.data<float>(), Y->template mutable_data<float>(), alpha_);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool ThresholdedReluGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n CAFFE_ENFORCE_GT(Y.numel(), 0);\n CAFFE_ENFORCE_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n ThresholdedReluGradientKernel<<<\n CAFFE_GET_BLOCKS(Y.numel()),\n CAFFE_CUDA_NUM_THREADS,\n 0,\n context_.cuda_stream()>>>(\n Y.numel(),\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_CUDA_OPERATOR(ThresholdedRelu, ThresholdedReluOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n ThresholdedReluGradient,\n ThresholdedReluGradientOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",808 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/thresholded_relu_op.h\"\n\nnamespace caffe2 {\nnamespace {\ntemplate <typename T>\n__global__ void ThresholdedReluKernel(const int N, const T* X, T* Y, T alpha_) {\n HIP_1D_KERNEL_LOOP(i, N) {\n Y[i] = X[i] > alpha_ ? X[i] : 0;\n }\n}\n\ntemplate <typename T>\n__global__ void\nThresholdedReluGradientKernel(const int N, const T* Y, const T* dY, T* dX) {\n HIP_1D_KERNEL_LOOP(i, N) {\n dX[i] = Y[i] > 0 ? dY[i] : 0;\n }\n}\n} // namespace\n\ntemplate <>\nbool ThresholdedReluOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n\n CAFFE_ENFORCE_GT(X.numel(), 0);\n auto* Y = Output(0, X.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( ThresholdedReluKernel), \n dim3(CAFFE_GET_BLOCKS(X.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n X.numel(), X.data<float>(), Y->template mutable_data<float>(), alpha_);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\ntemplate <>\nbool ThresholdedReluGradientOp<float, HIPContext>::RunOnDevice() {\n auto& Y = Input(0);\n auto& dY = Input(1);\n\n CAFFE_ENFORCE_GT(Y.numel(), 0);\n CAFFE_ENFORCE_EQ(dY.numel(), Y.numel());\n auto* dX = Output(0, Y.sizes(), at::dtype<float>());\n hipLaunchKernelGGL(( ThresholdedReluGradientKernel), \n dim3(CAFFE_GET_BLOCKS(Y.numel())),\n dim3(CAFFE_HIP_NUM_THREADS),\n 0,\n context_.hip_stream(), \n Y.numel(),\n Y.data<float>(),\n dY.data<float>(),\n dX->template mutable_data<float>());\n C10_HIP_KERNEL_LAUNCH_CHECK();\n\n return true;\n}\n\nREGISTER_HIP_OPERATOR(ThresholdedRelu, ThresholdedReluOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n ThresholdedReluGradient,\n ThresholdedReluGradientOp<float, HIPContext>);\n} // namespace caffe2\n###"809 },810 {811 "cuda": "\n#include \"caffe2/operators/transpose_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Transpose, TransposeOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",812 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/transpose_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Transpose, TransposeOp<HIPContext>);\n\n} // namespace caffe2\n###"813 },814 {815 "cuda": "\n\n#include \"caffe2/operators/unique_ops.h\"\n#include <thrust/device_vector.h>\n#include <thrust/sequence.h>\n#include <thrust/sort.h>\n#include <thrust/system/cuda/execution_policy.h>\n#include <thrust/unique.h>\n#include <thrust/version.h>\n#include \"caffe2/core/context_gpu.h\"\nnamespace caffe2 {\n#if THRUST_VERSION >= 100800\nnamespace {\n__global__ void remap_kernel(\n thrust::device_ptr<int> second_order, thrust::device_ptr<int> order, int* output, int N, int K) {\n int i = blockDim.x * blockIdx.x + threadIdx.x;\n if (i >= K)\n return;\n int idx = second_order[i];\n output[order[idx]] = i;\n \n for (idx++; idx < N && (i == K - 1 || idx != second_order[i + 1]); idx++) {\n output[order[idx]] = i;\n }\n return;\n}\n} \ntemplate <>\ntemplate <typename T>\nbool UniqueOp<CUDAContext>::DoRunWithType() {\n auto& inputTensor = Input(0);\n \n int N = inputTensor.dim32(0);\n CAFFE_ENFORCE_EQ(inputTensor.dim(), 1, \"Input should be a vector\");\n int* remapping = nullptr;\n if (REMAPPING < OutputSize()) {\n auto* remappingTensor =\n Output(REMAPPING, inputTensor.sizes(), at::dtype<int>());\n remapping = remappingTensor->template mutable_data<int>();\n }\n if (N <= 0) {\n \n Output(UNIQUE, {0}, at::dtype<T>());\n return true;\n }\n const T* input = inputTensor.template data<T>();\n ReinitializeTensor(&thrust_unique_buffer_, {N}, at::dtype<T>().device(CUDA));\n auto* buffer = thrust_unique_buffer_.template mutable_data<T>();\n context_.CopyItemsSameDevice(inputTensor.meta(), N, input, buffer);\n \n thrust::device_vector<int> order1(N), order2(N);\n thrust::sequence(\n thrust::cuda::par.on(context_.cuda_stream()), order1.begin(), order1.end());\n thrust::sequence(\n thrust::cuda::par.on(context_.cuda_stream()), order2.begin(), order2.end());\n \n \n \n \n \n \n \n thrust::sort_by_key(\n thrust::cuda::par.on(context_.cuda_stream()), buffer, buffer + N, order1.begin());\n \n \n \n \n \n \n auto new_last = thrust::unique_by_key(\n thrust::cuda::par.on(context_.cuda_stream()), buffer, buffer + N, order2.begin());\n int K = new_last.first - buffer;\n auto* uniqueTensor = Output(UNIQUE, {K}, at::dtype<T>());\n T* unique = uniqueTensor->template mutable_data<T>();\n context_.CopyItemsSameDevice(thrust_unique_buffer_.meta(), K, buffer, unique);\n \n \n \n \n if (remapping != nullptr) {\n \n remap_kernel<<<\n CAFFE_GET_BLOCKS(K), CAFFE_CUDA_NUM_THREADS, 0, context_.cuda_stream()>>>(\n order2.data(), order1.data(), remapping, N, K);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n }\n return true;\n}\nREGISTER_CUDA_OPERATOR(Unique, UniqueOp<CUDAContext>);\n#endif \n} \n\n###",816 "hip": " \n#include \"hip/hip_runtime.h\"\n\n#include \"caffe2/operators/unique_ops.h\"\n#include <thrust/device_vector.h>\n#include <thrust/sequence.h>\n#include <thrust/sort.h>\n#include <thrust/system/hip/execution_policy.h>\n#include <thrust/unique.h>\n#include <thrust/version.h>\n#include \"caffe2/core/hip/context_gpu.h\"\nnamespace caffe2 {\n#if THRUST_VERSION >= 100800\nnamespace {\n__global__ void remap_kernel(\n thrust::device_ptr<int> second_order, thrust::device_ptr<int> order, int* output, int N, int K) {\n int i = blockDim.x * blockIdx.x + threadIdx.x;\n if (i >= K)\n return;\n int idx = second_order[i];\n output[order[idx]] = i;\n \n for (idx++; idx < N && (i == K - 1 || idx != second_order[i + 1]); idx++) {\n output[order[idx]] = i;\n }\n return;\n}\n} \ntemplate <>\ntemplate <typename T>\nbool UniqueOp<HIPContext>::DoRunWithType() {\n auto& inputTensor = Input(0);\n \n int N = inputTensor.dim32(0);\n CAFFE_ENFORCE_EQ(inputTensor.dim(), 1, \"Input should be a vector\");\n int* remapping = nullptr;\n if (REMAPPING < OutputSize()) {\n auto* remappingTensor =\n Output(REMAPPING, inputTensor.sizes(), at::dtype<int>());\n remapping = remappingTensor->template mutable_data<int>();\n }\n if (N <= 0) {\n \n Output(UNIQUE, {0}, at::dtype<T>());\n return true;\n }\n const T* input = inputTensor.template data<T>();\n ReinitializeTensor(&thrust_unique_buffer_, {N}, at::dtype<T>().device(HIP));\n auto* buffer = thrust_unique_buffer_.template mutable_data<T>();\n context_.CopyItemsSameDevice(inputTensor.meta(), N, input, buffer);\n \n thrust::device_vector<int> order1(N), order2(N);\n thrust::sequence(\n thrust::hip::par.on(context_.hip_stream()), order1.begin(), order1.end());\n thrust::sequence(\n thrust::hip::par.on(context_.hip_stream()), order2.begin(), order2.end());\n \n \n \n \n \n \n \n thrust::sort_by_key(\n thrust::hip::par.on(context_.hip_stream()), buffer, buffer + N, order1.begin());\n \n \n \n \n \n \n auto new_last = thrust::unique_by_key(\n thrust::hip::par.on(context_.hip_stream()), buffer, buffer + N, order2.begin());\n int K = new_last.first - buffer;\n auto* uniqueTensor = Output(UNIQUE, {K}, at::dtype<T>());\n T* unique = uniqueTensor->template mutable_data<T>();\n context_.CopyItemsSameDevice(thrust_unique_buffer_.meta(), K, buffer, unique);\n \n \n \n \n if (remapping != nullptr) {\n \n hipLaunchKernelGGL(( remap_kernel), dim3(CAFFE_GET_BLOCKS(K)), dim3(CAFFE_HIP_NUM_THREADS), 0, context_.hip_stream(), order2.data(), order1.data(), remapping, N, K);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n }\n return true;\n}\nREGISTER_HIP_OPERATOR(Unique, UniqueOp<HIPContext>);\n#endif \n} ###"817 },818 {819 "cuda": "\n#include \"caffe2/operators/unsafe_coalesce.h\"\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(UnsafeCoalesce, UnsafeCoalesceOp<CUDAContext>);\n\n}\n\n\n###",820 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/unsafe_coalesce.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(UnsafeCoalesce, UnsafeCoalesceOp<HIPContext>);\n\n}\n###"821 },822 {823 "cuda": "\n#include <iostream>\n\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/core/flags.h\"\n#include \"caffe2/operators/utility_ops.h\"\n#include <gtest/gtest.h>\n\nC10_DECLARE_string(caffe_test_root);\n\nnamespace caffe2 {\n\nstatic void AddConstInput(\n const vector<int64_t>& shape,\n const float value,\n const string& name,\n Workspace* ws) {\n DeviceOption option;\n option.set_device_type(PROTO_CUDA);\n CUDAContext context(option);\n Blob* blob = ws->CreateBlob(name);\n auto* tensor = BlobGetMutableTensor(blob, CUDA);\n tensor->Resize(shape);\n math::Set<float, CUDAContext>(\n tensor->numel(), value, tensor->template mutable_data<float>(), &context);\n return;\n}\n\nTEST(UtilityOpGPUTest, testReshapeWithScalar) {\n if (!HasCudaGPU())\n return;\n Workspace ws;\n OperatorDef def;\n def.set_name(\"test_reshape\");\n def.set_type(\"Reshape\");\n def.add_input(\"X\");\n def.add_output(\"XNew\");\n def.add_output(\"OldShape\");\n def.add_arg()->CopyFrom(MakeArgument(\"shape\", vector<int64_t>{1}));\n def.mutable_device_option()->set_device_type(PROTO_CUDA);\n AddConstInput(vector<int64_t>(), 3.14, \"X\", &ws);\n // execute the op\n unique_ptr<OperatorBase> op(CreateOperator(def, &ws));\n EXPECT_TRUE(op->Run());\n Blob* XNew = ws.GetBlob(\"XNew\");\n const Tensor& XNewTensor = XNew->Get<Tensor>();\n EXPECT_EQ(1, XNewTensor.dim());\n EXPECT_EQ(1, XNewTensor.numel());\n}\n\n} // namespace caffe2\n\n\n###",824 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <iostream>\n\n#include \"caffe2/core/context.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/core/flags.h\"\n#include \"caffe2/operators/utility_ops.h\"\n#include <gtest/gtest.h>\n\nC10_DECLARE_string(caffe_test_root);\n\nnamespace caffe2 {\n\nstatic void AddConstInput(\n const vector<int64_t>& shape,\n const float value,\n const string& name,\n Workspace* ws) {\n DeviceOption option;\n option.set_device_type(PROTO_HIP);\n HIPContext context(option);\n Blob* blob = ws->CreateBlob(name);\n auto* tensor = BlobGetMutableTensor(blob, HIP);\n tensor->Resize(shape);\n math::Set<float, HIPContext>(\n tensor->numel(), value, tensor->template mutable_data<float>(), &context);\n return;\n}\n\nTEST(UtilityOpGPUTest, testReshapeWithScalar) {\n if (!HasHipGPU())\n return;\n Workspace ws;\n OperatorDef def;\n def.set_name(\"test_reshape\");\n def.set_type(\"Reshape\");\n def.add_input(\"X\");\n def.add_output(\"XNew\");\n def.add_output(\"OldShape\");\n def.add_arg()->CopyFrom(MakeArgument(\"shape\", vector<int64_t>{1}));\n def.mutable_device_option()->set_device_type(PROTO_HIP);\n AddConstInput(vector<int64_t>(), 3.14, \"X\", &ws);\n // execute the op\n unique_ptr<OperatorBase> op(CreateOperator(def, &ws));\n EXPECT_TRUE(op->Run());\n Blob* XNew = ws.GetBlob(\"XNew\");\n const Tensor& XNewTensor = XNew->Get<Tensor>();\n EXPECT_EQ(1, XNewTensor.dim());\n EXPECT_EQ(1, XNewTensor.numel());\n}\n\n} // namespace caffe2\n###"825 },826 {827 "cuda": "\n#include \"caffe2/operators/while_op.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(While, WhileOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",828 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/operators/while_op.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(While, WhileOp<HIPContext>);\n\n} // namespace caffe2\n###"829 },830 {831 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/zero_gradient_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(ZeroGradient, ZeroGradientOp<CUDAContext>);\n}\n\n\n###",832 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/zero_gradient_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(ZeroGradient, ZeroGradientOp<HIPContext>);\n}\n###"833 },834 {835 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/rnn/recurrent_network_blob_fetcher_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(\n RecurrentNetworkBlobFetcher,\n RecurrentNetworkBlobFetcherOp<CUDAContext>);\n} // namespace caffe2\n\n\n###",836 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/rnn/recurrent_network_blob_fetcher_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(\n RecurrentNetworkBlobFetcher,\n RecurrentNetworkBlobFetcherOp<HIPContext>);\n} // namespace caffe2\n###"837 },838 {839 "cuda": "\n#ifndef CAFFE2_OPERATORS_RECURRENT_NETWORK_GPU_EXECUTOR_H_\n#define CAFFE2_OPERATORS_RECURRENT_NETWORK_GPU_EXECUTOR_H_\n\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/rnn/recurrent_network_executor.h\"\n\n\n#include <map>\n\nnamespace caffe2 {\n\nclass CUDARecurrentNetworkExecutor : public RecurrentNetworkExecutorBase {\n public:\n CUDARecurrentNetworkExecutor(\n const NetDef& step_net_def,\n std::map<string, string>& recurrent_input_map,\n std::string timestep_blob)\n : RecurrentNetworkExecutorBase(step_net_def, recurrent_input_map, timestep_blob) {}\n\n ~CUDARecurrentNetworkExecutor();\n\n protected:\n bool Run(int T) override;\n\n bool RunBackwards(int T) override;\n\n bool ignoreLinkDependencies() override {\n return true;\n }\n\n void AnalyzeOps() override {\n /**\n * Check if there is an op that only depends on ops from previous\n * timestep, and that ops is not the last op. Then we can start computation\n * in subsequent timesteps before the whole previous timestep has finished.\n * If there is no parallelism, we can avoid overhead of event-based\n * dependency management.\n */\n has_timestep_parallelism_ = false;\n for (auto& rnn_op : timestep_ops_template_) {\n int i = rnn_op.order;\n if (rnn_op.parents.size() >= 1 && i < timestep_ops_template_.size() - 1) {\n bool only_recurrent_deps = std::all_of(\n rnn_op.parents.begin(),\n rnn_op.parents.end(), [&](const int &parent) {\n return parent > i;\n }\n );\n if (only_recurrent_deps) {\n VLOG(1) << \"Timestep parallel op: \" << ProtoDebugString(step_net_def_.op(i));\n has_timestep_parallelism_ = true;\n\n for (int dep : rnn_op.parents) {\n if (dep == timestep_ops_template_.size() - 1) {\n // This op depends on the last op of the previous iteration,\n // so it will block any parallelism\n has_timestep_parallelism_ = false;\n break;\n }\n }\n break;\n }\n }\n }\n LOG(INFO) << \"Analyzed ops for timestep parallelism: \" << has_timestep_parallelism_;\n }\n\n public:\n\n void setMaxStreams(int n) {\n max_cuda_streams_ = n;\n }\n\n private:\n void _ExecRange(int from, int to);\n\n std::vector<cudaEvent_t> events_;\n bool has_timestep_parallelism_ = false;\n int max_cuda_streams_ = 2;\n};\n}\n#endif\n\n\n###",840 "hip": " // !!! This is a file automatically generated by hipify!!!\n#ifndef CAFFE2_OPERATORS_RECURRENT_NETWORK_GPU_EXECUTOR_H_\n#define CAFFE2_OPERATORS_RECURRENT_NETWORK_GPU_EXECUTOR_H_\n\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/rnn/recurrent_network_executor.h\"\n\n\n#include <map>\n\nnamespace caffe2 {\n\nclass HIPRecurrentNetworkExecutor : public RecurrentNetworkExecutorBase {\n public:\n HIPRecurrentNetworkExecutor(\n const NetDef& step_net_def,\n std::map<string, string>& recurrent_input_map,\n std::string timestep_blob)\n : RecurrentNetworkExecutorBase(step_net_def, recurrent_input_map, timestep_blob) {}\n\n ~HIPRecurrentNetworkExecutor();\n\n protected:\n bool Run(int T) override;\n\n bool RunBackwards(int T) override;\n\n bool ignoreLinkDependencies() override {\n return true;\n }\n\n void AnalyzeOps() override {\n /**\n * Check if there is an op that only depends on ops from previous\n * timestep, and that ops is not the last op. Then we can start computation\n * in subsequent timesteps before the whole previous timestep has finished.\n * If there is no parallelism, we can avoid overhead of event-based\n * dependency management.\n */\n has_timestep_parallelism_ = false;\n for (auto& rnn_op : timestep_ops_template_) {\n int i = rnn_op.order;\n if (rnn_op.parents.size() >= 1 && i < timestep_ops_template_.size() - 1) {\n bool only_recurrent_deps = std::all_of(\n rnn_op.parents.begin(),\n rnn_op.parents.end(), [&](const int &parent) {\n return parent > i;\n }\n );\n if (only_recurrent_deps) {\n VLOG(1) << \"Timestep parallel op: \" << ProtoDebugString(step_net_def_.op(i));\n has_timestep_parallelism_ = true;\n\n for (int dep : rnn_op.parents) {\n if (dep == timestep_ops_template_.size() - 1) {\n // This op depends on the last op of the previous iteration,\n // so it will block any parallelism\n has_timestep_parallelism_ = false;\n break;\n }\n }\n break;\n }\n }\n }\n LOG(INFO) << \"Analyzed ops for timestep parallelism: \" << has_timestep_parallelism_;\n }\n\n public:\n\n void setMaxStreams(int n) {\n max_hip_streams_ = n;\n }\n\n private:\n void _ExecRange(int from, int to);\n\n std::vector<hipEvent_t> events_;\n bool has_timestep_parallelism_ = false;\n int max_hip_streams_ = 2;\n};\n}\n#endif\n###"841 },842 {843 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <ATen/cuda/ApplyGridUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\ntemplate <typename scalar_t>\nvoid threshold_kernel_impl(\n TensorIteratorBase& iter,\n scalar_t threshold,\n scalar_t value) {\n gpu_kernel_with_scalars(\n iter, [=] GPU_LAMBDA(scalar_t x, scalar_t other) -> scalar_t {\n return x <= threshold ? value : other;\n });\n}\n\nstatic void threshold_kernel_cuda(\n TensorIteratorBase& iter,\n const Scalar& threshold,\n const Scalar& value) {\n AT_DISPATCH_ALL_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"threshold_cuda\",\n [&] {\n threshold_kernel_impl<scalar_t>(\n iter, threshold.to<scalar_t>(), value.to<scalar_t>());\n });\n}\n\n} // namespace\n\nREGISTER_DISPATCH(threshold_stub, &threshold_kernel_cuda);\n\n} // namespace at::native\n\n\n###",844 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#define _USE_MATH_DEFINES\n\n#include <ATen/native/Activation.h>\n\n#include <cmath>\n\n#include <thrust/tuple.h>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/core/TensorBase.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <ATen/hip\\ApplyGridUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\nnamespace at::native {\nnamespace {\n\ntemplate <typename scalar_t>\nvoid threshold_kernel_impl(\n TensorIteratorBase& iter,\n scalar_t threshold,\n scalar_t value) {\n gpu_kernel_with_scalars(\n iter, [=] GPU_LAMBDA(scalar_t x, scalar_t other) -> scalar_t {\n return x <= threshold ? value : other;\n });\n}\n\nstatic void threshold_kernel_hip(\n TensorIteratorBase& iter,\n const Scalar& threshold,\n const Scalar& value) {\n AT_DISPATCH_ALL_TYPES_AND2(\n at::ScalarType::Half,\n at::ScalarType::BFloat16,\n iter.dtype(),\n \"threshold_hip\",\n [&] {\n threshold_kernel_impl<scalar_t>(\n iter, threshold.to<scalar_t>(), value.to<scalar_t>());\n });\n}\n\n} // namespace\n\nREGISTER_DISPATCH(threshold_stub, &threshold_kernel_hip);\n\n} // namespace at::native\n###"845 },846 {847 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/operators/rnn/recurrent_network_op.h\"\n\nnamespace caffe2 {\n\nnamespace detail {\n\ntemplate <typename T, typename Context>\nvoid initializeRecurrentInput(\n const RecurrentInput& rc,\n int32_t seqLen,\n int32_t batchSize,\n Workspace* ws,\n Context* context);\n\nnamespace {\n\ntemplate <typename T>\n__global__\nvoid initRecurrentInput_kernel(\n size_t stateSize,\n const T* input,\n T* state) {\n // index into appropriate target buffer\n const int block_id = blockIdx.x;\n T* state_local = state + block_id*stateSize;\n\n // copy\n for (int idx=threadIdx.x; idx < stateSize; idx+=blockDim.x) {\n state_local[idx] = input[idx];\n }\n}\n\n\n}; // namespace\n\ntemplate <>\nvoid repeatCopy(\n size_t repeat_n,\n size_t n,\n const float* src,\n float* dst,\n CUDAContext* context) {\n initRecurrentInput_kernel<float><<<repeat_n, CAFFE_CUDA_NUM_THREADS, 0, context->cuda_stream()>>>(\n n, src, dst);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\ntemplate <>\nvoid repeatCopy(\n size_t repeat_n,\n size_t n,\n const at::Half* src,\n at::Half* dst,\n CUDAContext* context) {\n initRecurrentInput_kernel<at::Half><<<repeat_n, CAFFE_CUDA_NUM_THREADS, 0, context->cuda_stream()>>>(\n n, src, dst);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\n}; // namespace detail\n\ntemplate <>\nbool RecurrentNetworkOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\ntemplate <>\nbool RecurrentNetworkGradientOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\ntemplate <>\nbool AccumulateInputGradientOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(1));\n}\n\ntemplate <>\nbool RNNApplyLinkOp<CUDAContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(1));\n}\n\nREGISTER_CUDA_OPERATOR(\n RecurrentNetwork,\n RecurrentNetworkOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n RecurrentNetworkGradient,\n RecurrentNetworkGradientOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n rnn_internal_accumulate_gradient_input,\n AccumulateInputGradientOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n rnn_internal_apply_link,\n RNNApplyLinkOp<CUDAContext>);\n\n\n} // namespace caffe2\n\n\n###",848 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/operators/rnn/recurrent_network_op.h\"\n\nnamespace caffe2 {\n\nnamespace detail {\n\ntemplate <typename T, typename Context>\nvoid initializeRecurrentInput(\n const RecurrentInput& rc,\n int32_t seqLen,\n int32_t batchSize,\n Workspace* ws,\n Context* context);\n\nnamespace {\n\ntemplate <typename T>\n__global__\nvoid initRecurrentInput_kernel(\n size_t stateSize,\n const T* input,\n T* state) {\n // index into appropriate target buffer\n const int block_id = blockIdx.x;\n T* state_local = state + block_id*stateSize;\n\n // copy\n for (int idx=threadIdx.x; idx < stateSize; idx+=blockDim.x) {\n state_local[idx] = input[idx];\n }\n}\n\n\n}; // namespace\n\ntemplate <>\nvoid repeatCopy(\n size_t repeat_n,\n size_t n,\n const float* src,\n float* dst,\n HIPContext* context) {\n hipLaunchKernelGGL(( initRecurrentInput_kernel<float>), dim3(repeat_n), dim3(CAFFE_HIP_NUM_THREADS), 0, context->hip_stream(), \n n, src, dst);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\ntemplate <>\nvoid repeatCopy(\n size_t repeat_n,\n size_t n,\n const at::Half* src,\n at::Half* dst,\n HIPContext* context) {\n hipLaunchKernelGGL(( initRecurrentInput_kernel<at::Half>), dim3(repeat_n), dim3(CAFFE_HIP_NUM_THREADS), 0, context->hip_stream(), \n n, src, dst);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\n}; // namespace detail\n\ntemplate <>\nbool RecurrentNetworkOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\ntemplate <>\nbool RecurrentNetworkGradientOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(0));\n}\n\ntemplate <>\nbool AccumulateInputGradientOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(1));\n}\n\ntemplate <>\nbool RNNApplyLinkOp<HIPContext>::RunOnDevice() {\n return DispatchHelper<TensorTypes<float, at::Half>>::call(this, Input(1));\n}\n\nREGISTER_HIP_OPERATOR(\n RecurrentNetwork,\n RecurrentNetworkOp<HIPContext>);\nREGISTER_HIP_OPERATOR(\n RecurrentNetworkGradient,\n RecurrentNetworkGradientOp<HIPContext>);\nREGISTER_HIP_OPERATOR(\n rnn_internal_accumulate_gradient_input,\n AccumulateInputGradientOp<HIPContext>);\nREGISTER_HIP_OPERATOR(\n rnn_internal_apply_link,\n RNNApplyLinkOp<HIPContext>);\n\n\n} // namespace caffe2\n###"849 },850 {851 "cuda": "\n#include \"caffe2/queue/queue_ops.h\"\n#include \"caffe2/utils/math.h\"\n\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(CreateBlobsQueue, CreateBlobsQueueOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(EnqueueBlobs, EnqueueBlobsOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(DequeueBlobs, DequeueBlobsOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(CloseBlobsQueue, CloseBlobsQueueOp<CUDAContext>);\n\nREGISTER_CUDA_OPERATOR(SafeEnqueueBlobs, SafeEnqueueBlobsOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(SafeDequeueBlobs, SafeDequeueBlobsOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",852 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/queue/queue_ops.h\"\n#include \"caffe2/utils/math.h\"\n\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(CreateBlobsQueue, CreateBlobsQueueOp<HIPContext>);\nREGISTER_HIP_OPERATOR(EnqueueBlobs, EnqueueBlobsOp<HIPContext>);\nREGISTER_HIP_OPERATOR(DequeueBlobs, DequeueBlobsOp<HIPContext>);\nREGISTER_HIP_OPERATOR(CloseBlobsQueue, CloseBlobsQueueOp<HIPContext>);\n\nREGISTER_HIP_OPERATOR(SafeEnqueueBlobs, SafeEnqueueBlobsOp<HIPContext>);\nREGISTER_HIP_OPERATOR(SafeDequeueBlobs, SafeDequeueBlobsOp<HIPContext>);\n\n} // namespace caffe2\n###"853 },854 {855 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/sgd/iter_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(Iter, IterOp<CUDAContext>);\nREGISTER_CUDA_OPERATOR(AtomicIter, AtomicIterOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",856 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/sgd/iter_op.h\"\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(Iter, IterOp<HIPContext>);\nREGISTER_HIP_OPERATOR(AtomicIter, AtomicIterOp<HIPContext>);\n\n} // namespace caffe2\n###"857 },858 {859 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/sgd/lars_op.h\"\n\nnamespace caffe2 {\n__global__ void ComputeLearningRateKernel(\n const float* wd,\n const float* trust,\n const float* lr_max,\n float offset,\n float lr_min,\n float* X_norm,\n float* dX_norm,\n float* lr_rescaled) {\n float val = 1.0;\n\n if (*X_norm > 0) {\n val = (*trust) / (*dX_norm / *X_norm + (*wd) + offset);\n }\n *lr_rescaled = fmaxf(fminf(val, *lr_max), lr_min);\n}\n\ntemplate <>\nvoid LarsOp<float, CUDAContext>::ComputeLearningRate(\n const float* wd,\n const float* trust,\n const float* lr_max,\n float offset,\n float lr_min,\n float* X_norm,\n float* dX_norm,\n float* lr_rescaled) {\n ComputeLearningRateKernel<<<1, 1, 0, context_.cuda_stream()>>>(\n wd, trust, lr_max, offset, lr_min, X_norm, dX_norm, lr_rescaled);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_CUDA_OPERATOR(Lars, LarsOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",860 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/sgd/lars_op.h\"\n\nnamespace caffe2 {\n__global__ void ComputeLearningRateKernel(\n const float* wd,\n const float* trust,\n const float* lr_max,\n float offset,\n float lr_min,\n float* X_norm,\n float* dX_norm,\n float* lr_rescaled) {\n float val = 1.0;\n\n if (*X_norm > 0) {\n val = (*trust) / (*dX_norm / *X_norm + (*wd) + offset);\n }\n *lr_rescaled = fmaxf(fminf(val, *lr_max), lr_min);\n}\n\ntemplate <>\nvoid LarsOp<float, HIPContext>::ComputeLearningRate(\n const float* wd,\n const float* trust,\n const float* lr_max,\n float offset,\n float lr_min,\n float* X_norm,\n float* dX_norm,\n float* lr_rescaled) {\n hipLaunchKernelGGL(( ComputeLearningRateKernel), dim3(1), dim3(1), 0, context_.hip_stream(), \n wd, trust, lr_max, offset, lr_min, X_norm, dX_norm, lr_rescaled);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\nREGISTER_HIP_OPERATOR(Lars, LarsOp<float, HIPContext>);\n} // namespace caffe2\n###"861 },862 {863 "cuda": "\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/sgd/learning_rate_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(LearningRate, LearningRateOp<float, CUDAContext>);\n} // namespace caffe2\n\n\n###",864 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/sgd/learning_rate_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(LearningRate, LearningRateOp<float, HIPContext>);\n} // namespace caffe2\n###"865 },866 {867 "cuda": "\n#include \"caffe2/sgd/rmsprop_op.h\"\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n\nnamespace caffe2 {\n\n__global__ void RmsPropUpdate(\n int N,\n const float* g,\n const float* ms,\n const float* mom,\n float* ng,\n float* nms,\n float* nmom,\n float decay,\n float momentum,\n float epsilon,\n const float* lr) {\n CUDA_1D_KERNEL_LOOP(i, N) {\n // Update new mean square estimate\n nms[i] = ms[i] + (1.0f - decay) * (g[i] * g[i] - ms[i]);\n // Update momentum estimate\n nmom[i] =\n mom[i] * momentum + lr[0] * g[i] / sqrtf(epsilon + nms[i]);\n // New gradient is the momentum\n ng[i] = nmom[i];\n }\n}\n\ntemplate <>\nvoid rmsprop_update<CUDAContext>(\n int N,\n const float* g,\n const float* ms,\n const float* mom,\n float* ng,\n float* nms,\n float* nmom,\n float decay,\n float momentum,\n float epsilon,\n const float* lr,\n CUDAContext* context) {\n RmsPropUpdate<<<CAFFE_GET_BLOCKS(N), CAFFE_CUDA_NUM_THREADS, 0, context->cuda_stream()>>>(\n N, g, ms, mom, ng, nms, nmom, decay, momentum, epsilon, lr);\n C10_CUDA_KERNEL_LAUNCH_CHECK();\n}\n\n\nREGISTER_CUDA_OPERATOR(RmsProp, RmsPropOp<float, CUDAContext>);\n\n}\n\n\n###",868 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#include \"caffe2/sgd/rmsprop_op.h\"\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n\nnamespace caffe2 {\n\n__global__ void RmsPropUpdate(\n int N,\n const float* g,\n const float* ms,\n const float* mom,\n float* ng,\n float* nms,\n float* nmom,\n float decay,\n float momentum,\n float epsilon,\n const float* lr) {\n HIP_1D_KERNEL_LOOP(i, N) {\n // Update new mean square estimate\n nms[i] = ms[i] + (1.0f - decay) * (g[i] * g[i] - ms[i]);\n // Update momentum estimate\n nmom[i] =\n mom[i] * momentum + lr[0] * g[i] / sqrtf(epsilon + nms[i]);\n // New gradient is the momentum\n ng[i] = nmom[i];\n }\n}\n\ntemplate <>\nvoid rmsprop_update<HIPContext>(\n int N,\n const float* g,\n const float* ms,\n const float* mom,\n float* ng,\n float* nms,\n float* nmom,\n float decay,\n float momentum,\n float epsilon,\n const float* lr,\n HIPContext* context) {\n hipLaunchKernelGGL(( RmsPropUpdate), dim3(CAFFE_GET_BLOCKS(N)), dim3(CAFFE_HIP_NUM_THREADS), 0, context->hip_stream(), \n N, g, ms, mom, ng, nms, nmom, decay, momentum, epsilon, lr);\n C10_HIP_KERNEL_LAUNCH_CHECK();\n}\n\n\nREGISTER_HIP_OPERATOR(RmsProp, RmsPropOp<float, HIPContext>);\n\n}\n###"869 },870 {871 "cuda": "\n#include \"caffe2/core/common_gpu.h\"\n#include \"caffe2/core/context_gpu.h\"\n#include \"caffe2/sgd/weight_scale_op.h\"\n\nnamespace caffe2 {\nREGISTER_CUDA_OPERATOR(WeightScale, WeightScaleOp<CUDAContext>);\n\ntemplate <typename T>\nvoid weight_scale_update_kernel(\n int N,\n const T* w,\n const T& scale,\n int64_t iter,\n int64_t stepsize,\n int64_t update_upper_bound,\n T* nw,\n CUDAContext* context) {\n const auto w_size = N * sizeof(float);\n if (iter % stepsize != 0 || iter >= update_upper_bound) {\n (void)cudaMemcpy(nw, w, w_size, cudaMemcpyDefault);\n } else {\n // perform the weight scaling\n caffe2::math::Scale<T, T, CUDAContext>(N, scale, w, nw, context);\n }\n}\n\ntemplate <>\ntemplate <typename T>\nbool WeightScaleOp<CUDAContext>::DoRunWithType() {\n const auto iter =\n OperatorBase::Input<Tensor>(ITER, CPU).template data<int64_t>()[0] + 1;\n weight_scale_update_kernel<T>(\n Input(WEIGHTS).size(),\n Input(WEIGHTS).template data<T>(),\n scale_,\n iter,\n stepsize_,\n update_upper_bound_,\n Output(OUTPUT_WEIGHTS)->template mutable_data<T>(),\n &context_);\n return true;\n}\n\n} // namespace caffe2\n\n\n###",872 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"caffe2/core/hip/common_gpu.h\"\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"caffe2/sgd/weight_scale_op.h\"\n\nnamespace caffe2 {\nREGISTER_HIP_OPERATOR(WeightScale, WeightScaleOp<HIPContext>);\n\ntemplate <typename T>\nvoid weight_scale_update_kernel(\n int N,\n const T* w,\n const T& scale,\n int64_t iter,\n int64_t stepsize,\n int64_t update_upper_bound,\n T* nw,\n HIPContext* context) {\n const auto w_size = N * sizeof(float);\n if (iter % stepsize != 0 || iter >= update_upper_bound) {\n (void)hipMemcpy(nw, w, w_size, hipMemcpyDefault);\n } else {\n // perform the weight scaling\n caffe2::math::Scale<T, T, HIPContext>(N, scale, w, nw, context);\n }\n}\n\ntemplate <>\ntemplate <typename T>\nbool WeightScaleOp<HIPContext>::DoRunWithType() {\n const auto iter =\n OperatorBase::Input<Tensor>(ITER, CPU).template data<int64_t>()[0] + 1;\n weight_scale_update_kernel<T>(\n Input(WEIGHTS).size(),\n Input(WEIGHTS).template data<T>(),\n scale_,\n iter,\n stepsize_,\n update_upper_bound_,\n Output(OUTPUT_WEIGHTS)->template mutable_data<T>(),\n &context_);\n return true;\n}\n\n} // namespace caffe2\n###"873 },874 {875 "cuda": "\n#pragma once\n\n// cub sort support for CUB_WRAPPED_NAMESPACE is added to cub 1.13.1 in:\n// https://github.com/NVIDIA/cub/pull/326\n// CUB_WRAPPED_NAMESPACE is defined globally in cmake/Dependencies.cmake\n// starting from CUDA 11.5\n#if defined(CUB_WRAPPED_NAMESPACE) || defined(THRUST_CUB_WRAPPED_NAMESPACE)\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() true\n#else\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() false\n#endif\n\n#if USE_GLOBAL_CUB_WRAPPED_NAMESPACE()\nnamespace caffe2 {\nnamespace cub = ::CUB_WRAPPED_NAMESPACE::cub;\n}\n#endif\n\n\n###",876 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n// cub sort support for CUB_WRAPPED_NAMESPACE is added to cub 1.13.1 in:\n// https://github.com/NVIDIA/cub/pull/326\n// CUB_WRAPPED_NAMESPACE is defined globally in cmake/Dependencies.cmake\n// starting from HIP 11.5\n#if defined(CUB_WRAPPED_NAMESPACE) || defined(THRUST_CUB_WRAPPED_NAMESPACE)\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() true\n#else\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() false\n#endif\n\n#if USE_GLOBAL_CUB_WRAPPED_NAMESPACE()\nnamespace caffe2 {\nnamespace cub = ::CUB_WRAPPED_NAMESPACE::cub;\n}\n#endif\n###"877 },878 {879 "cuda": "\n#ifndef CAFFE2_UTILS_GPU_ATOMICS_H_\n#define CAFFE2_UTILS_GPU_ATOMICS_H_\n\n#include <cuda_runtime.h>\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\ninline __device__ void gpu_atomic_add(T* address, const T val) {\n atomicAdd(address, val);\n}\n\ntemplate <>\ninline __device__ void gpu_atomic_add(float* address, const float val) {\n#if defined(USE_ROCM) && defined(__gfx908__)\n atomicAddNoRet(address, val);\n#else\n atomicAdd(address, val);\n#endif\n}\n\n} // namespace\n\n} // namespace caffe2\n\n#endif // CAFFE2_UTILS_GPU_ATOMICS_H_\n\n\n###",880 "hip": " // !!! This is a file automatically generated by hipify!!!\n#ifndef CAFFE2_UTILS_GPU_ATOMICS_H_\n#define CAFFE2_UTILS_GPU_ATOMICS_H_\n\n#include <hip/hip_runtime.h>\n\nnamespace caffe2 {\n\nnamespace {\n\ntemplate <typename T>\ninline __device__ void gpu_atomic_add(T* address, const T val) {\n atomicAdd(address, val);\n}\n\ntemplate <>\ninline __device__ void gpu_atomic_add(float* address, const float val) {\n#if defined(USE_ROCM) && defined(__gfx908__)\n atomicAddNoRet(address, val);\n#else\n atomicAdd(address, val);\n#endif\n}\n\n} // namespace\n\n} // namespace caffe2\n\n#endif // CAFFE2_UTILS_GPU_ATOMICS_H_\n###"881 },882 {883 "cuda": "\n#ifndef CAFFE2_UTILS_GPU_SCAN_UTILS_H_\n#define CAFFE2_UTILS_GPU_SCAN_UTILS_H_\n#include \"caffe2/utils/GpuDefs.cuh\"\nnamespace caffe2 {\n\n\n\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void inclusivePrefixScan(T* smem, T in, T* out, BinaryFunction binop) {\n \n \n smem[threadIdx.x] = in;\n __syncthreads();\n for (int offset = 1; offset < blockDim.x; offset *= 2) {\n T val = 0;\n if (threadIdx.x >= offset) {\n val = binop(smem[threadIdx.x - offset], smem[threadIdx.x]);\n }\n __syncthreads();\n if (threadIdx.x >= offset) {\n smem[threadIdx.x] = val;\n }\n __syncthreads();\n }\n *out = smem[threadIdx.x];\n \n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void exclusivePrefixScan(T* smem, T in, T* out, T* carry, BinaryFunction binop) {\n \n \n inclusivePrefixScan<T, false, BinaryFunction>(smem, in, out, binop);\n *out -= in;\n *carry = smem[blockDim.x - 1];\n \n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void inclusiveBinaryPrefixScan(T* smem, bool in, T* out, BinaryFunction binop) {\n \n#if defined(USE_ROCM)\n unsigned long long int vote = __ballot(in);\n T index = __popcll(getLaneMaskLe() & vote);\n T carry = __popcll(vote);\n#else\n T vote = __ballot_sync(__activemask(), in);\n T index = __popc(getLaneMaskLe() & vote);\n T carry = __popc(vote);\n#endif \n int warp = threadIdx.x / kWarpSize;\n \n if (getLaneId() == 0) {\n smem[warp] = carry;\n }\n __syncthreads();\n \n \n if (threadIdx.x == 0) {\n int current = 0;\n for (int i = 0; i < blockDim.x / kWarpSize; ++i) {\n T v = smem[i];\n smem[i] = binop(smem[i], current);\n current = binop(current, v);\n }\n }\n __syncthreads();\n \n if (warp >= 1) {\n index = binop(index, smem[warp - 1]);\n }\n *out = index;\n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void exclusiveBinaryPrefixScan(T* smem, bool in, T* out, T* carry, BinaryFunction binop) {\n inclusiveBinaryPrefixScan<T, false, BinaryFunction>(smem, in, out, binop);\n \n *out -= (T) in;\n \n#if defined(USE_ROCM)\n *carry = smem[math::DivUp<int>(blockDim.x, kWarpSize) - 1];\n#else\n *carry = smem[(blockDim.x / kWarpSize) - 1];\n#endif \n if (KillWARDependency) {\n __syncthreads();\n }\n}\n} \n#endif \n\n###",884 "hip": " \n#include \"hip/hip_runtime.h\"\n#ifndef CAFFE2_UTILS_GPU_SCAN_UTILS_H_\n#define CAFFE2_UTILS_GPU_SCAN_UTILS_H_\n#include \"caffe2/utils/hip/GpuDefs.cuh\"\nnamespace caffe2 {\n\n\n\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void inclusivePrefixScan(T* smem, T in, T* out, BinaryFunction binop) {\n \n \n smem[threadIdx.x] = in;\n __syncthreads();\n for (int offset = 1; offset < blockDim.x; offset *= 2) {\n T val = 0;\n if (threadIdx.x >= offset) {\n val = binop(smem[threadIdx.x - offset], smem[threadIdx.x]);\n }\n __syncthreads();\n if (threadIdx.x >= offset) {\n smem[threadIdx.x] = val;\n }\n __syncthreads();\n }\n *out = smem[threadIdx.x];\n \n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void exclusivePrefixScan(T* smem, T in, T* out, T* carry, BinaryFunction binop) {\n \n \n inclusivePrefixScan<T, false, BinaryFunction>(smem, in, out, binop);\n *out -= in;\n *carry = smem[blockDim.x - 1];\n \n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void inclusiveBinaryPrefixScan(T* smem, bool in, T* out, BinaryFunction binop) {\n \n#if defined(USE_ROCM)\n unsigned long long int vote = __ballot(in);\n T index = __popcll(getLaneMaskLe() & vote);\n T carry = __popcll(vote);\n#else\n T vote = __ballot_sync(__activemask(), in);\n T index = __popc(getLaneMaskLe() & vote);\n T carry = __popc(vote);\n#endif \n int warp = threadIdx.x / kWarpSize;\n \n if (getLaneId() == 0) {\n smem[warp] = carry;\n }\n __syncthreads();\n \n \n if (threadIdx.x == 0) {\n int current = 0;\n for (int i = 0; i < blockDim.x / kWarpSize; ++i) {\n T v = smem[i];\n smem[i] = binop(smem[i], current);\n current = binop(current, v);\n }\n }\n __syncthreads();\n \n if (warp >= 1) {\n index = binop(index, smem[warp - 1]);\n }\n *out = index;\n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void exclusiveBinaryPrefixScan(T* smem, bool in, T* out, T* carry, BinaryFunction binop) {\n inclusiveBinaryPrefixScan<T, false, BinaryFunction>(smem, in, out, binop);\n \n *out -= (T) in;\n \n#if defined(USE_ROCM)\n *carry = smem[math::DivUp<int>(blockDim.x, kWarpSize) - 1];\n#else\n *carry = smem[(blockDim.x / kWarpSize) - 1];\n#endif \n if (KillWARDependency) {\n __syncthreads();\n }\n}\n} \n#endif ###"885 },886 {887 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_CUDA char airy_ai_name[] = \"airy_ai_forward\";\n\nvoid airy_ai_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"airy_ai_cuda\", [&]() {\n jitted_gpu_kernel<airy_ai_name, scalar_t, scalar_t, 1>(iterator, airy_ai_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"airy_ai_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return airy_ai_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(special_airy_ai_stub, &airy_ai_kernel_cuda);\n} // namespace at::native\n\n\n###",888 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_HIP char airy_ai_name[] = \"airy_ai_forward\";\n\nvoid airy_ai_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"airy_ai_hip\", [&]() {\n jitted_gpu_kernel<airy_ai_name, scalar_t, scalar_t, 1>(iterator, airy_ai_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"airy_ai_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return airy_ai_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(special_airy_ai_stub, &airy_ai_kernel_hip);\n} // namespace at::native\n###"889 },890 {891 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAConfig.h>\n#include <ATen/cuda/cub.cuh>\nnamespace at {\nnamespace cuda {\nnamespace cub {\nnamespace detail {\ntemplate <typename key_t, int value_size>\nvoid radix_sort_pairs_impl(\n const key_t* keys_in, key_t* keys_out, const OpaqueType<value_size>* values_in, OpaqueType<value_size>* values_out, int64_t n, bool descending, int64_t begin_bit, int64_t end_bit) {\n TORCH_CHECK(\n n <= std::numeric_limits<int>::max(), \"cub sort does not support sorting more than INT_MAX elements\");\n using key_t_ = typename detail::cuda_type<key_t>::type;\n auto allocator = c10::cuda::CUDACachingAllocator::get();\n c10::DataPtr keys_out_owner;\n if (keys_out == nullptr) {\n keys_out_owner = allocator->allocate(n * sizeof(key_t));\n keys_out = reinterpret_cast<key_t*>(keys_out_owner.get());\n }\n const key_t_* keys_in_ = reinterpret_cast<const key_t_*>(keys_in);\n key_t_* keys_out_ = reinterpret_cast<key_t_*>(keys_out);\n if (descending) {\n CUB_WRAPPER(\n NO_ROCM(at_cuda_detail)::cub::DeviceRadixSort::SortPairsDescending, keys_in_, keys_out_, values_in, values_out, n, begin_bit, end_bit, c10::cuda::getCurrentCUDAStream());\n } else {\n CUB_WRAPPER(\n NO_ROCM(at_cuda_detail)::cub::DeviceRadixSort::SortPairs, keys_in_, keys_out_, values_in, values_out, n, begin_bit, end_bit, c10::cuda::getCurrentCUDAStream());\n }\n}\n#define AT_INSTANTIATE_SORT_PAIRS(key_t, value_size) template void radix_sort_pairs_impl( const key_t* keys_in, key_t* keys_out, const OpaqueType<value_size>* values_in, OpaqueType<value_size>* values_out, int64_t n, bool descending, int64_t begin_bit, int64_t end_bit);\nAT_INSTANTIATE_SORT_PAIRS(int32_t, 1)\nAT_INSTANTIATE_SORT_PAIRS(int32_t, 2)\nAT_INSTANTIATE_SORT_PAIRS(int32_t, 4)\nAT_INSTANTIATE_SORT_PAIRS(int64_t, 1)\nAT_INSTANTIATE_SORT_PAIRS(int64_t, 2)\nAT_INSTANTIATE_SORT_PAIRS(int64_t, 4)\n#define AT_INSTANTIATE_SORT_PAIRS_8(scalar_t, ScalarType) AT_INSTANTIATE_SORT_PAIRS(scalar_t, 8)\nAT_FORALL_SCALAR_TYPES_AND2(Bool, Half, AT_INSTANTIATE_SORT_PAIRS_8)\n\n#if !AT_ROCM_ENABLED() || (AT_ROCM_ENABLED() && ROCM_VERSION >= 40500)\nAT_INSTANTIATE_SORT_PAIRS(c10::BFloat16, 8)\n#endif\n} \n} \n} \n} \n\n###",892 "hip": " \n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPConfig.h>\n#include <ATen/hip\\cub.cuh>\nnamespace at {\nnamespace hip {\nnamespace cub {\nnamespace detail {\ntemplate <typename key_t, int value_size>\nvoid radix_sort_pairs_impl(\n const key_t* keys_in, key_t* keys_out, const OpaqueType<value_size>* values_in, OpaqueType<value_size>* values_out, int64_t n, bool descending, int64_t begin_bit, int64_t end_bit) {\n TORCH_CHECK(\n n <= std::numeric_limits<int>::max(), \"cub sort does not support sorting more than INT_MAX elements\");\n using key_t_ = typename detail::hip_type<key_t>::type;\n auto allocator = c10::hip::HIPCachingAllocator::get();\n c10::DataPtr keys_out_owner;\n if (keys_out == nullptr) {\n keys_out_owner = allocator->allocate(n * sizeof(key_t));\n keys_out = reinterpret_cast<key_t*>(keys_out_owner.get());\n }\n const key_t_* keys_in_ = reinterpret_cast<const key_t_*>(keys_in);\n key_t_* keys_out_ = reinterpret_cast<key_t_*>(keys_out);\n if (descending) {\n CUB_WRAPPER(\n NO_ROCM(at_hip_detail)::hipcub::DeviceRadixSort::SortPairsDescending, keys_in_, keys_out_, values_in, values_out, n, begin_bit, end_bit, c10::hip::getCurrentHIPStream());\n } else {\n CUB_WRAPPER(\n NO_ROCM(at_hip_detail)::hipcub::DeviceRadixSort::SortPairs, keys_in_, keys_out_, values_in, values_out, n, begin_bit, end_bit, c10::hip::getCurrentHIPStream());\n }\n}\n#define AT_INSTANTIATE_SORT_PAIRS(key_t, value_size) template void radix_sort_pairs_impl( const key_t* keys_in, key_t* keys_out, const OpaqueType<value_size>* values_in, OpaqueType<value_size>* values_out, int64_t n, bool descending, int64_t begin_bit, int64_t end_bit);\nAT_INSTANTIATE_SORT_PAIRS(int32_t, 1)\nAT_INSTANTIATE_SORT_PAIRS(int32_t, 2)\nAT_INSTANTIATE_SORT_PAIRS(int32_t, 4)\nAT_INSTANTIATE_SORT_PAIRS(int64_t, 1)\nAT_INSTANTIATE_SORT_PAIRS(int64_t, 2)\nAT_INSTANTIATE_SORT_PAIRS(int64_t, 4)\n#define AT_INSTANTIATE_SORT_PAIRS_8(scalar_t, ScalarType) AT_INSTANTIATE_SORT_PAIRS(scalar_t, 8)\nAT_FORALL_SCALAR_TYPES_AND2(Bool, Half, AT_INSTANTIATE_SORT_PAIRS_8)\n\n#if !AT_ROCM_ENABLED() || (AT_ROCM_ENABLED() && ROCM_VERSION >= 40500)\nAT_INSTANTIATE_SORT_PAIRS(c10::BFloat16, 8)\n#endif\n} \n} \n} \n} ###"893 },894 {895 "cuda": "\n#ifndef CAFFE2_UTILS_MATH_REDUCE_CUH_\n#define CAFFE2_UTILS_MATH_REDUCE_CUH_\n#include \"caffe2/utils/cub_namespace.cuh\"\n#include <cub/block/block_reduce.cuh>\n#include \"caffe2/core/common_gpu.h\"\nnamespace caffe2 {\ntemplate <typename T>\nusing BlockReduce = cub::BlockReduce<T, CAFFE_CUDA_NUM_THREADS>;\ntemplate <typename T, int kBlockDimX, int kBlockDimY>\nusing BlockReduce2D = cub::\n BlockReduce<T, kBlockDimX, cub::BLOCK_REDUCE_WARP_REDUCTIONS, kBlockDimY>;\n#define DISPATCH_REDUCE_KERNEL_BY_2D_BLOCK_WITH_TYPE_1( size, Func, T, grid_dim, cuda_stream, ...) do { if (size >= 128) { Func<T, 1, 128> <<<grid_dim, dim3(1, 128), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } else if (size >= 64) { Func<T, 2, 64><<<grid_dim, dim3(2, 64), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } else if (size >= 32) { Func<T, 4, 32><<<grid_dim, dim3(4, 32), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } else { Func<T, 8, 16><<<grid_dim, dim3(8, 16), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } } while (false)\n#define DISPATCH_REDUCE_KERNEL_BY_2D_BLOCK_WITH_TYPE_2( size, Func, T1, T2, grid_dim, cuda_stream, ...) do { if (size >= 128) { Func<T1, T2, 1, 128> <<<grid_dim, dim3(1, 128), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } else if (size >= 64) { Func<T1, T2, 2, 64> <<<grid_dim, dim3(2, 64), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } else if (size >= 32) { Func<T1, T2, 4, 32> <<<grid_dim, dim3(4, 32), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } else { Func<T1, T2, 8, 16> <<<grid_dim, dim3(8, 16), 0, cuda_stream>>>(__VA_ARGS__); C10_CUDA_KERNEL_LAUNCH_CHECK(); } } while (false)\n} \n#endif \n\n###",896 "hip": " \n#include \"hip/hip_runtime.h\"\n#ifndef CAFFE2_UTILS_MATH_REDUCE_CUH_\n#define CAFFE2_UTILS_MATH_REDUCE_CUH_\n#include \"caffe2/utils/cub_namespace.cuh\"\n#include <hipcub/hipcub.hpp>\n#include \"caffe2/core/hip/common_gpu.h\"\nnamespace caffe2 {\ntemplate <typename T>\nusing BlockReduce = hipcub::BlockReduce<T, CAFFE_HIP_NUM_THREADS>;\ntemplate <typename T, int kBlockDimX, int kBlockDimY>\nusing BlockReduce2D = hipcub::\n BlockReduce<T, kBlockDimX, hipcub::BLOCK_REDUCE_WARP_REDUCTIONS, kBlockDimY>;\n#define DISPATCH_REDUCE_KERNEL_BY_2D_BLOCK_WITH_TYPE_1( size, Func, T, grid_dim, hip_stream, ...) do { if (size >= 128) { hipLaunchKernelGGL(( Func<T, 1, 128>) , dim3(grid_dim), dim3(dim3(1, 128)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } else if (size >= 64) { hipLaunchKernelGGL(( Func<T, 2, 64>), dim3(grid_dim), dim3(dim3(2, 64)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } else if (size >= 32) { hipLaunchKernelGGL(( Func<T, 4, 32>), dim3(grid_dim), dim3(dim3(4, 32)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } else { hipLaunchKernelGGL(( Func<T, 8, 16>), dim3(grid_dim), dim3(dim3(8, 16)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } } while (false)\n#define DISPATCH_REDUCE_KERNEL_BY_2D_BLOCK_WITH_TYPE_2( size, Func, T1, T2, grid_dim, hip_stream, ...) do { if (size >= 128) { hipLaunchKernelGGL(( Func<T1, T2, 1, 128>) , dim3(grid_dim), dim3(dim3(1, 128)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } else if (size >= 64) { hipLaunchKernelGGL(( Func<T1, T2, 2, 64>) , dim3(grid_dim), dim3(dim3(2, 64)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } else if (size >= 32) { hipLaunchKernelGGL(( Func<T1, T2, 4, 32>) , dim3(grid_dim), dim3(dim3(4, 32)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } else { hipLaunchKernelGGL(( Func<T1, T2, 8, 16>) , dim3(grid_dim), dim3(dim3(8, 16)), 0, hip_stream, __VA_ARGS__); C10_HIP_KERNEL_LAUNCH_CHECK(); } } while (false)\n} \n#endif ###"897 },898 {899 "cuda": "\n#include <caffe2/core/common_gpu.h>\n#include <caffe2/core/context_gpu.h>\n#include <caffe2/video/video_input_op.h>\n\nnamespace caffe2 {\n\nREGISTER_CUDA_OPERATOR(VideoInput, VideoInputOp<CUDAContext>);\n\n} // namespace caffe2\n\n\n###",900 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <caffe2/core/hip/common_gpu.h>\n#include <caffe2/core/hip/context_gpu.h>\n#include <caffe2/video/video_input_op.h>\n\nnamespace caffe2 {\n\nREGISTER_HIP_OPERATOR(VideoInput, VideoInputOp<HIPContext>);\n\n} // namespace caffe2\n###"901 },902 {903 "cuda": "\n\n\n#include <cfloat>\n#include \"caffe2/core/context_gpu.h\"\n#include \"modules/detectron/sample_as_op.h\"\n#include <stdio.h>\nnamespace caffe2 {\ntemplate <>\nbool SampleAsOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0); \n auto& L = Input(1); \n CAFFE_ENFORCE(\n X.dim32(0) == L.dim32(0), \"X.dim32(0) must be equal to L.dim32(0)\", \"(\", X.dim32(0), \" vs. \", L.dim32(0), \")\");\n \n std::vector<int> labels(L.dim32(0));\n context_.CopyBytes<CUDAContext, CPUContext>(\n L.dim32(0) * sizeof(int), L.data<int>(), &labels[0]);\n \n context_.FinishDeviceComputation();\n int count = 0;\n for (int i = 0; i < L.dim32(0); i++) {\n if (labels[i] > 0) {\n count++;\n }\n }\n assert(count > 0);\n \n vector<int64_t> out_shape(X.sizes().vec());\n out_shape[0] = count;\n auto* Y = Output(0, out_shape, at::dtype<float>()); \n const int len = X.size() / X.dim32(0);\n float* output = Y->mutable_data<float>();\n for (int i = 0; i < L.dim32(0); i++) {\n if (labels[i] > 0) {\n context_.CopyBytes<CUDAContext, CUDAContext>(\n len * sizeof(float), X.data<float>() + i * len, output);\n output += len;\n } \n } \n return true;\n}\ntemplate <>\nbool SampleAsGradientOp<float, CUDAContext>::RunOnDevice() {\n auto& X = Input(0);\n auto& L = Input(1);\n auto& dY = Input(2);\n auto* dX = Output(0, X.sizes(), at::dtype<float>());\n \n std::vector<int> labels(L.dim32(0));\n context_.CopyBytes<CUDAContext, CPUContext>(\n L.dim32(0) * sizeof(int), L.data<int>(), &labels[0]);\n \n context_.FinishDeviceComputation();\n \n math::Set<float, CUDAContext>(\n dX->size(), 0.f, dX->mutable_data<float>(), &context_);\n const int len = X.size() / X.dim32(0);\n const float* input = dY.data<float>();\n for (int i = 0; i < L.dim32(0); i++) {\n if (labels[i] > 0) {\n context_.CopyBytes<CUDAContext, CUDAContext>(\n len * sizeof(float), input, dX->mutable_data<float>() + i * len);\n input += len;\n } \n } \n return true;\n}\nREGISTER_CUDA_OPERATOR(SampleAs, SampleAsOp<float, CUDAContext>);\nREGISTER_CUDA_OPERATOR(\n SampleAsGradient, SampleAsGradientOp<float, CUDAContext>);\n} \n\n###",904 "hip": " \n\n\n#include <cfloat>\n#include \"caffe2/core/hip/context_gpu.h\"\n#include \"modules/detectron/sample_as_op.h\"\n#include <stdio.h>\nnamespace caffe2 {\ntemplate <>\nbool SampleAsOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0); \n auto& L = Input(1); \n CAFFE_ENFORCE(\n X.dim32(0) == L.dim32(0), \"X.dim32(0) must be equal to L.dim32(0)\", \"(\", X.dim32(0), \" vs. \", L.dim32(0), \")\");\n \n std::vector<int> labels(L.dim32(0));\n context_.CopyBytes<HIPContext, CPUContext>(\n L.dim32(0) * sizeof(int), L.data<int>(), &labels[0]);\n \n context_.FinishDeviceComputation();\n int count = 0;\n for (int i = 0; i < L.dim32(0); i++) {\n if (labels[i] > 0) {\n count++;\n }\n }\n assert(count > 0);\n \n vector<int64_t> out_shape(X.sizes().vec());\n out_shape[0] = count;\n auto* Y = Output(0, out_shape, at::dtype<float>()); \n const int len = X.size() / X.dim32(0);\n float* output = Y->mutable_data<float>();\n for (int i = 0; i < L.dim32(0); i++) {\n if (labels[i] > 0) {\n context_.CopyBytes<HIPContext, HIPContext>(\n len * sizeof(float), X.data<float>() + i * len, output);\n output += len;\n } \n } \n return true;\n}\ntemplate <>\nbool SampleAsGradientOp<float, HIPContext>::RunOnDevice() {\n auto& X = Input(0);\n auto& L = Input(1);\n auto& dY = Input(2);\n auto* dX = Output(0, X.sizes(), at::dtype<float>());\n \n std::vector<int> labels(L.dim32(0));\n context_.CopyBytes<HIPContext, CPUContext>(\n L.dim32(0) * sizeof(int), L.data<int>(), &labels[0]);\n \n context_.FinishDeviceComputation();\n \n math::Set<float, HIPContext>(\n dX->size(), 0.f, dX->mutable_data<float>(), &context_);\n const int len = X.size() / X.dim32(0);\n const float* input = dY.data<float>();\n for (int i = 0; i < L.dim32(0); i++) {\n if (labels[i] > 0) {\n context_.CopyBytes<HIPContext, HIPContext>(\n len * sizeof(float), input, dX->mutable_data<float>() + i * len);\n input += len;\n } \n } \n return true;\n}\nREGISTER_HIP_OPERATOR(SampleAs, SampleAsOp<float, HIPContext>);\nREGISTER_HIP_OPERATOR(\n SampleAsGradient, SampleAsGradientOp<float, HIPContext>);\n} ###"905 },906 {907 "cuda": "\n// Copyright (c) Meta Platforms, Inc. and affiliates.\n//\n// This source code is licensed under the BSD-style license found in the\n// LICENSE file in the root directory of this source tree.\n\n#pragma once\n\n#include <ATen/ATen.h>\n#include <vector>\n\nnamespace torch {\nnamespace distributed {\nnamespace c10d {\nnamespace quantization {\n\nat::Tensor _float_to_bfloat16_cuda(const at::Tensor& input);\nat::Tensor _bfloat16_to_float_cuda(const at::Tensor& input);\n\n} // namespace quantization\n} // namespace c10d\n} // namespace distributed\n} // namespace torch\n\n\n###",908 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Copyright (c) Meta Platforms, Inc. and affiliates.\n//\n// This source code is licensed under the BSD-style license found in the\n// LICENSE file in the root directory of this source tree.\n\n#pragma once\n\n#include <ATen/ATen.h>\n#include <vector>\n\nnamespace torch {\nnamespace distributed {\nnamespace c10d {\nnamespace quantization {\n\nat::Tensor _float_to_bfloat16_hip(const at::Tensor& input);\nat::Tensor _bfloat16_to_float_hip(const at::Tensor& input);\n\n} // namespace quantization\n} // namespace c10d\n} // namespace distributed\n} // namespace torch\n###"909 },910 {911 "cuda": "\n\n#define __NVFUSER_BFLOAT_TO_US(var) *(reinterpret_cast<unsigned short*>(&(var)))\n#define __NVFUSER_BFLOAT_TO_CUS(var) \\\n *(reinterpret_cast<const unsigned short*>(&(var)))\n\nstruct __bfloat;\n__device__ __bfloat __float2bfloat(const float);\n\nstruct __align__(2) __bfloat {\n __bfloat() = default;\n\n __device__ __bfloat(const float f) {\n __x = __float2bfloat(f).__x;\n }\n\n protected:\n unsigned short __x;\n};\n\n__device__ __bfloat __float2bfloat(const float f) {\n __bfloat val;\n asm(\"{ cvt.rn.bf16.f32 %0, %1;}\\n\"\n : \"=h\"(__NVFUSER_BFLOAT_TO_US(val))\n : \"f\"(f));\n return val;\n}\n\n__device__ float __bfloat2float(const __bfloat h) {\n float val;\n asm(\"{ mov.b32 %0, {0,%1};}\\n\"\n : \"=f\"(val)\n : \"h\"(__NVFUSER_BFLOAT_TO_CUS(h)));\n return val;\n}\n\n\n###",912 "hip": " // !!! This is a file automatically generated by hipify!!!\n\n#define __NVFUSER_BFLOAT_TO_US(var) *(reinterpret_cast<unsigned short*>(&(var)))\n#define __NVFUSER_BFLOAT_TO_CUS(var) \\\n *(reinterpret_cast<const unsigned short*>(&(var)))\n\nstruct __bfloat;\n__device__ __bfloat __float2bfloat(const float);\n\nstruct __align__(2) __bfloat {\n __bfloat() = default;\n\n __device__ __bfloat(const float f) {\n __x = __float2bfloat(f).__x;\n }\n\n protected:\n unsigned short __x;\n};\n\n__device__ __bfloat __float2bfloat(const float f) {\n __bfloat val;\n asm(\"{ cvt.rn.bf16.f32 %0, %1;}\\n\"\n : \"=h\"(__NVFUSER_BFLOAT_TO_US(val))\n : \"f\"(f));\n return val;\n}\n\n__device__ float __bfloat2float(const __bfloat h) {\n float val;\n asm(\"{ mov.b32 %0, {0,%1};}\\n\"\n : \"=f\"(val)\n : \"h\"(__NVFUSER_BFLOAT_TO_CUS(h)));\n return val;\n}\n###"913 },914 {915 "cuda": "\n\nstruct __align__(2) __bfloat {\n __bfloat() = default;\n\n inline __device__ __bfloat(const float f) {\n if (f != f) {\n __x = uint16_t(0x7FC0);\n } else {\n union {\n uint32_t U32;\n float F32;\n };\n\n F32 = f;\n uint32_t rounding_bias = ((U32 >> 16) & 1) + uint32_t(0x7FFF);\n __x = static_cast<uint16_t>((U32 + rounding_bias) >> 16);\n }\n }\n\n inline __device__ operator float() const {\n float res = 0;\n uint32_t tmp = __x;\n tmp <<= 16;\n float* tempRes = reinterpret_cast<float*>(&tmp);\n res = *tempRes;\n return res;\n }\n\n protected:\n unsigned short __x;\n};\n\n__device__ __bfloat __float2bfloat(const float f) {\n return __bfloat(f);\n}\n\n__device__ float __bfloat2float(const __bfloat h) {\n return float(h);\n}\n\n\n###",916 "hip": " // !!! This is a file automatically generated by hipify!!!\n\nstruct __align__(2) __bfloat {\n __bfloat() = default;\n\n inline __device__ __bfloat(const float f) {\n if (f != f) {\n __x = uint16_t(0x7FC0);\n } else {\n union {\n uint32_t U32;\n float F32;\n };\n\n F32 = f;\n uint32_t rounding_bias = ((U32 >> 16) & 1) + uint32_t(0x7FFF);\n __x = static_cast<uint16_t>((U32 + rounding_bias) >> 16);\n }\n }\n\n inline __device__ operator float() const {\n float res = 0;\n uint32_t tmp = __x;\n tmp <<= 16;\n float* tempRes = reinterpret_cast<float*>(&tmp);\n res = *tempRes;\n return res;\n }\n\n protected:\n unsigned short __x;\n};\n\n__device__ __bfloat __float2bfloat(const float f) {\n return __bfloat(f);\n}\n\n__device__ float __bfloat2float(const __bfloat h) {\n return float(h);\n}\n###"917 },918 {919 "cuda": "\n\n// Default block synchronization. Just use __barrier_sync\nnamespace block_sync {\n\n__forceinline__ __device__ void init() {}\n\n// Thread-block synchronization\n__forceinline__ __device__ void sync() {\n __barrier_sync(0);\n}\n\n} // namespace block_sync\n\n\n###",920 "hip": " // !!! This is a file automatically generated by hipify!!!\n\n// Default block synchronization. Just use __barrier_sync\nnamespace block_sync {\n\n__forceinline__ __device__ void init() {}\n\n// Thread-block synchronization\n__forceinline__ __device__ void sync() {\n __barrier_sync(0);\n}\n\n} // namespace block_sync\n###"921 },922 {923 "cuda": "\n\n#define __NVFUSER_HALF_TO_US(var) *(reinterpret_cast<unsigned short*>(&(var)))\n#define __NVFUSER_HALF_TO_CUS(var) \\\n *(reinterpret_cast<const unsigned short*>(&(var)))\n\nstruct __half;\n__device__ __half __float2half(const float);\n\nstruct __align__(2) __half {\n __half() = default;\n\n __device__ __half(const float f) {\n __x = __float2half(f).__x;\n }\n\n protected:\n unsigned short __x;\n};\n\n__device__ __half __float2half(const float f) {\n __half val;\n asm(\"{ cvt.rn.f16.f32 %0, %1;}\\n\"\n : \"=h\"(__NVFUSER_HALF_TO_US(val))\n : \"f\"(f));\n return val;\n}\n\n__device__ float __half2float(const __half h) {\n float val;\n asm(\"{ cvt.f32.f16 %0, %1;}\\n\" : \"=f\"(val) : \"h\"(__NVFUSER_HALF_TO_CUS(h)));\n return val;\n}\n\n__device__ __half __double2half(const double d) {\n#if __CUDA_ARCH__ >= 700\n __half val;\n asm(\"{ cvt.rn.f16.f64 %0, %1;}\\n\"\n : \"=h\"(__NVFUSER_HALF_TO_US(val))\n : \"d\"(d));\n return val;\n#else\n return __float2half(static_cast<float>(d));\n#endif\n}\n\n__device__ double __half2double(const __half h) {\n#if __CUDA_ARCH__ >= 700\n double val;\n asm(\"{ cvt.f64.f16 %0, %1;}\\n\" : \"=d\"(val) : \"h\"(__NVFUSER_HALF_TO_CUS(h)));\n return val;\n#else\n return static_cast<double>(__half2float(h));\n#endif\n}\n\n\n###",924 "hip": " // !!! This is a file automatically generated by hipify!!!\n\n#define __NVFUSER_HALF_TO_US(var) *(reinterpret_cast<unsigned short*>(&(var)))\n#define __NVFUSER_HALF_TO_CUS(var) \\\n *(reinterpret_cast<const unsigned short*>(&(var)))\n\nstruct __half;\n__device__ __half __float2half(const float);\n\nstruct __align__(2) __half {\n __half() = default;\n\n __device__ __half(const float f) {\n __x = __float2half(f).__x;\n }\n\n protected:\n unsigned short __x;\n};\n\n__device__ __half __float2half(const float f) {\n __half val;\n asm(\"{ cvt.rn.f16.f32 %0, %1;}\\n\"\n : \"=h\"(__NVFUSER_HALF_TO_US(val))\n : \"f\"(f));\n return val;\n}\n\n__device__ float __half2float(const __half h) {\n float val;\n asm(\"{ cvt.f32.f16 %0, %1;}\\n\" : \"=f\"(val) : \"h\"(__NVFUSER_HALF_TO_CUS(h)));\n return val;\n}\n\n__device__ __half __double2half(const double d) {\n#if __HIP_ARCH__ >= 700\n __half val;\n asm(\"{ cvt.rn.f16.f64 %0, %1;}\\n\"\n : \"=h\"(__NVFUSER_HALF_TO_US(val))\n : \"d\"(d));\n return val;\n#else\n return __float2half(static_cast<float>(d));\n#endif\n}\n\n__device__ double __half2double(const __half h) {\n#if __HIP_ARCH__ >= 700\n double val;\n asm(\"{ cvt.f64.f16 %0, %1;}\\n\" : \"=d\"(val) : \"h\"(__NVFUSER_HALF_TO_CUS(h)));\n return val;\n#else\n return static_cast<double>(__half2float(h));\n#endif\n}\n###"925 },926 {927 "cuda": "\nnamespace fused_reduction {\n\n\n\n\n\n\n\ntemplate <\n int NumVals, typename DataTypeT, typename IndexTypeT, template <int, typename>\n typename MakeTuple>\nstruct WelfordTripletTuple {\n static constexpr int num_vals = NumVals;\n using DataType = DataTypeT;\n using IndexType = IndexTypeT;\n using DataTuple = typename MakeTuple<NumVals, DataType>::type;\n using IndexTuple = typename MakeTuple<NumVals, IndexType>::type;\n DataTuple avg;\n DataTuple var;\n IndexTuple N;\n WelfordTripletTuple(\n const DataTuple& avg, const DataTuple& var, const IndexTuple& N)\n : avg(avg), var(var), N(N) {}\n};\ntemplate <int NumVals, typename DataType, typename IndexType>\nusing LocalWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataType, IndexType, MakeLocalTuple>;\ntemplate <int NumVals, typename DataType, typename IndexType>\nusing RefWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataType, IndexType, MakeRefTuple>;\ntemplate <int NumVals, typename DataType, typename IndexType>\nusing ConstRefWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataType, IndexType, MakeConstRefTuple>;\ntemplate <int NumVals, typename DataTypeT, typename IndexTypeT>\nusing VolatilePtrWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataTypeT, IndexTypeT, MakeVolatilePtrTuple>;\n\n\ntemplate <typename WelfordTripletTupleType>\n__inline__ __device__ static void operator+=(\n WelfordTripletTupleType& triplet, nvfuser_index_t offset) {\n triplet.avg += offset;\n triplet.var += offset;\n triplet.N += offset;\n}\n\ntemplate <typename DstType, typename SrcType>\n__inline__ __device__ static void copyWelfordTripletTuple(\n DstType& dst, nvfuser_index_t dst_offset, const SrcType& src, nvfuser_index_t src_offset = 0) {\n copyTuple(dst.avg, dst_offset, src.avg, src_offset);\n copyTuple(dst.var, dst_offset, src.var, src_offset);\n copyTuple(dst.N, dst_offset, src.N, src_offset);\n}\n\ntemplate <typename DstType, typename SrcType>\n__inline__ __device__ static void copyWelfordTripletTuple(\n DstType& dst, const SrcType& src, nvfuser_index_t src_offset = 0) {\n copyWelfordTripletTuple(dst, 0, src, src_offset);\n}\n\ntemplate <typename DstType, typename SrcType, typename PredType>\n__inline__ __device__ static void copyWelfordTripletTupleIf(\n DstType& dst, const SrcType& src, const PredType& pred) {\n copyTupleIf(dst.avg, src.avg, pred);\n copyTupleIf(dst.var, src.var, pred);\n copyTupleIf(dst.N, src.N, pred);\n}\n} \n\n###",928 "hip": " \nnamespace fused_reduction {\n\n\n\n\n\n\n\ntemplate <\n int NumVals, typename DataTypeT, typename IndexTypeT, template <int, typename>\n typename MakeTuple>\nstruct WelfordTripletTuple {\n static constexpr int num_vals = NumVals;\n using DataType = DataTypeT;\n using IndexType = IndexTypeT;\n using DataTuple = typename MakeTuple<NumVals, DataType>::type;\n using IndexTuple = typename MakeTuple<NumVals, IndexType>::type;\n DataTuple avg;\n DataTuple var;\n IndexTuple N;\n WelfordTripletTuple(\n const DataTuple& avg, const DataTuple& var, const IndexTuple& N)\n : avg(avg), var(var), N(N) {}\n};\ntemplate <int NumVals, typename DataType, typename IndexType>\nusing LocalWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataType, IndexType, MakeLocalTuple>;\ntemplate <int NumVals, typename DataType, typename IndexType>\nusing RefWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataType, IndexType, MakeRefTuple>;\ntemplate <int NumVals, typename DataType, typename IndexType>\nusing ConstRefWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataType, IndexType, MakeConstRefTuple>;\ntemplate <int NumVals, typename DataTypeT, typename IndexTypeT>\nusing VolatilePtrWelfordTripletTuple =\n WelfordTripletTuple<NumVals, DataTypeT, IndexTypeT, MakeVolatilePtrTuple>;\n\n\ntemplate <typename WelfordTripletTupleType>\n__inline__ __device__ static void operator+=(\n WelfordTripletTupleType& triplet, nvfuser_index_t offset) {\n triplet.avg += offset;\n triplet.var += offset;\n triplet.N += offset;\n}\n\ntemplate <typename DstType, typename SrcType>\n__inline__ __device__ static void copyWelfordTripletTuple(\n DstType& dst, nvfuser_index_t dst_offset, const SrcType& src, nvfuser_index_t src_offset = 0) {\n copyTuple(dst.avg, dst_offset, src.avg, src_offset);\n copyTuple(dst.var, dst_offset, src.var, src_offset);\n copyTuple(dst.N, dst_offset, src.N, src_offset);\n}\n\ntemplate <typename DstType, typename SrcType>\n__inline__ __device__ static void copyWelfordTripletTuple(\n DstType& dst, const SrcType& src, nvfuser_index_t src_offset = 0) {\n copyWelfordTripletTuple(dst, 0, src, src_offset);\n}\n\ntemplate <typename DstType, typename SrcType, typename PredType>\n__inline__ __device__ static void copyWelfordTripletTupleIf(\n DstType& dst, const SrcType& src, const PredType& pred) {\n copyTupleIf(dst.avg, src.avg, pred);\n copyTupleIf(dst.var, src.var, pred);\n copyTupleIf(dst.N, src.N, pred);\n}\n} ###"929 },930 {931 "cuda": "\nnamespace grid_broadcast {\n\n// Broadcasts per-thread values across threads and blocks.\n//\n// Function parameters:\n// - out: Per-thread output location\n// - inp_val: Per-thread input value\n// - work_buf: Temporary buffer for communication across threads/blocks\n// - sync_flags: A vector of integers for synchronizations\n//\n// Template parameters:\n// - X/Y/Z_BLOCK: When true, broadcasts across thread blocks along the X/Y/Z\n// dimensions\n// - X/Y/Z_THREAD: When true, broadcasts across threads along the X/Y/Z\n// dimensions\ntemplate <\n bool X_BLOCK,\n bool Y_BLOCK,\n bool Z_BLOCK,\n bool X_THREAD,\n bool Y_THREAD,\n bool Z_THREAD,\n typename T>\n__device__ void broadcast(\n T& out,\n const T& inp_val,\n volatile T* work_buf,\n Tensor<int64_t, 1> sync_flags,\n bool read_write_pred) {\n // Number of values broadcasted in the grid dimensions\n const auto grid_seg_size =\n index_utils::maskedSize<X_BLOCK, Y_BLOCK, Z_BLOCK>(gridDim);\n\n // Index of the broadcast we're performing out of the grid_seg_size\n const auto grid_seg_idx =\n index_utils::maskedOffset<!X_BLOCK, !Y_BLOCK, !Z_BLOCK>(\n blockIdx, gridDim);\n\n // Number of threads not participating in a broadcast dimension, this is the\n // number of thread entries to expect in the work buffer, therefore a striding\n const auto block_stride =\n index_utils::maskedSize<!X_THREAD, !Y_THREAD, !Z_THREAD>(blockDim);\n\n // Which broadcast in the block this is to line up the entry with the work\n // buffer\n const auto thread_offset =\n index_utils::maskedOffset<!X_THREAD, !Y_THREAD, !Z_THREAD>(\n threadIdx, blockDim);\n\n const bool has_valid_data = (!X_BLOCK || blockIdx.x == gridDim.x - 1) &&\n (!Y_BLOCK || blockIdx.y == gridDim.y - 1) &&\n (!Z_BLOCK || blockIdx.z == gridDim.z - 1) &&\n (!X_THREAD || threadIdx.x == 0) && (!Y_THREAD || threadIdx.y == 0) &&\n (!Z_THREAD || threadIdx.z == 0);\n\n if (has_valid_data && read_write_pred) {\n work_buf[grid_seg_idx * block_stride + thread_offset] = inp_val;\n __threadfence();\n }\n\n grid_sync::sync<X_BLOCK, Y_BLOCK, Z_BLOCK, true>(\n sync_flags[grid_seg_idx], grid_seg_size);\n\n if (read_write_pred) {\n out = work_buf[grid_seg_idx * block_stride + thread_offset];\n }\n\n // Make sure everyone has read from the buffer before continuing the kernel\n // and potentially overwriting\n grid_sync::sync<X_BLOCK, Y_BLOCK, Z_BLOCK, true>(\n sync_flags[grid_seg_idx], grid_seg_size);\n}\n} // namespace grid_broadcast\n\n\n###",932 "hip": " // !!! This is a file automatically generated by hipify!!!\nnamespace grid_broadcast {\n\n// Broadcasts per-thread values across threads and blocks.\n//\n// Function parameters:\n// - out: Per-thread output location\n// - inp_val: Per-thread input value\n// - work_buf: Temporary buffer for communication across threads/blocks\n// - sync_flags: A vector of integers for synchronizations\n//\n// Template parameters:\n// - X/Y/Z_BLOCK: When true, broadcasts across thread blocks along the X/Y/Z\n// dimensions\n// - X/Y/Z_THREAD: When true, broadcasts across threads along the X/Y/Z\n// dimensions\ntemplate <\n bool X_BLOCK,\n bool Y_BLOCK,\n bool Z_BLOCK,\n bool X_THREAD,\n bool Y_THREAD,\n bool Z_THREAD,\n typename T>\n__device__ void broadcast(\n T& out,\n const T& inp_val,\n volatile T* work_buf,\n Tensor<int64_t, 1> sync_flags,\n bool read_write_pred) {\n // Number of values broadcasted in the grid dimensions\n const auto grid_seg_size =\n index_utils::maskedSize<X_BLOCK, Y_BLOCK, Z_BLOCK>(gridDim);\n\n // Index of the broadcast we're performing out of the grid_seg_size\n const auto grid_seg_idx =\n index_utils::maskedOffset<!X_BLOCK, !Y_BLOCK, !Z_BLOCK>(\n blockIdx, gridDim);\n\n // Number of threads not participating in a broadcast dimension, this is the\n // number of thread entries to expect in the work buffer, therefore a striding\n const auto block_stride =\n index_utils::maskedSize<!X_THREAD, !Y_THREAD, !Z_THREAD>(blockDim);\n\n // Which broadcast in the block this is to line up the entry with the work\n // buffer\n const auto thread_offset =\n index_utils::maskedOffset<!X_THREAD, !Y_THREAD, !Z_THREAD>(\n threadIdx, blockDim);\n\n const bool has_valid_data = (!X_BLOCK || blockIdx.x == gridDim.x - 1) &&\n (!Y_BLOCK || blockIdx.y == gridDim.y - 1) &&\n (!Z_BLOCK || blockIdx.z == gridDim.z - 1) &&\n (!X_THREAD || threadIdx.x == 0) && (!Y_THREAD || threadIdx.y == 0) &&\n (!Z_THREAD || threadIdx.z == 0);\n\n if (has_valid_data && read_write_pred) {\n work_buf[grid_seg_idx * block_stride + thread_offset] = inp_val;\n __threadfence();\n }\n\n grid_sync::sync<X_BLOCK, Y_BLOCK, Z_BLOCK, true>(\n sync_flags[grid_seg_idx], grid_seg_size);\n\n if (read_write_pred) {\n out = work_buf[grid_seg_idx * block_stride + thread_offset];\n }\n\n // Make sure everyone has read from the buffer before continuing the kernel\n // and potentially overwriting\n grid_sync::sync<X_BLOCK, Y_BLOCK, Z_BLOCK, true>(\n sync_flags[grid_seg_idx], grid_seg_size);\n}\n} // namespace grid_broadcast\n###"933 },934 {935 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_CUDA char bessel_j0_name[] = \"bessel_j0_forward\";\n\nvoid bessel_j0_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j0_cuda\", [&]() {\n jitted_gpu_kernel<bessel_j0_name, scalar_t, scalar_t, 1>(iterator, bessel_j0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j0_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_j0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(special_bessel_j0_stub, &bessel_j0_kernel_cuda);\n} // namespace at::native\n\n\n###",936 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_HIP char bessel_j0_name[] = \"bessel_j0_forward\";\n\nvoid bessel_j0_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j0_hip\", [&]() {\n jitted_gpu_kernel<bessel_j0_name, scalar_t, scalar_t, 1>(iterator, bessel_j0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j0_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_j0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(special_bessel_j0_stub, &bessel_j0_kernel_hip);\n} // namespace at::native\n###"937 },938 {939 "cuda": "\nnamespace grid_sync {\n\n#define FIRST_UINT64_BIT ((uint64_t)1 << (sizeof(uint64_t) * 8 - 1))\ntemplate <typename T>\n__device__ T globalAsVolatile(volatile T& global_val) {\n return global_val;\n}\n\n\n\n\n\n\n\n\n\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK, bool PERSISTENT>\n__device__ void sync(\n int64_t& semaphore, const uint64_t& segment_size, const bool last_block) {\n \n __threadfence();\n \n block_sync::sync();\n \n if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {\n \n \n \n \n uint64_t semaphore_increment = 1;\n \n \n \n if (last_block) {\n semaphore_increment = FIRST_UINT64_BIT - (segment_size - 1);\n }\n uint64_t oldArrive =\n atomicAdd(reinterpret_cast<uint64_t*>(&semaphore), semaphore_increment);\n \n \n \n unsigned int ns = 8;\n while ((PERSISTENT || last_block) &&\n ((oldArrive ^ globalAsVolatile(semaphore)) & FIRST_UINT64_BIT) ==\n 0) {\n \n \n#if __CUDA_ARCH__ >= 700\n \n __nanosleep(ns); \n if (ns < 256) {\n ns *= 2;\n }\n#endif\n }\n }\n \n block_sync::sync();\n}\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK, bool PERSISTENT>\n__device__ void sync(int64_t& semaphore, const uint64_t& segment_size) {\n sync<X_BLOCK, Y_BLOCK, Z_BLOCK, PERSISTENT>(\n semaphore, segment_size, index_utils::maskedIsLast<X_BLOCK, Y_BLOCK, Z_BLOCK>(blockIdx, gridDim));\n}\n\n\n\n\n\n\n\n\n\n\n\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK>\n__device__ void sync(\n int64_t& semaphore, const uint64_t& segment_size, const nvfuser_index_t n_entrances) {\n \n __threadfence();\n \n block_sync::sync();\n \n if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {\n \n \n \n bool last_block =\n index_utils::maskedIsLast<X_BLOCK, Y_BLOCK, Z_BLOCK>(blockIdx, gridDim);\n if (last_block) {\n int64_t finished_val =\n ((int64_t)(\n index_utils::maskedSize<X_BLOCK, Y_BLOCK, Z_BLOCK>(gridDim) -\n 1)) *\n ((int64_t)n_entrances);\n unsigned int ns = 8;\n \n while (globalAsVolatile(semaphore) < finished_val) {\n#if __CUDA_ARCH__ >= 700\n \n __nanosleep(ns); \n if (ns < 256) {\n ns *= 2;\n }\n#endif\n }\n } else {\n auto old = atomicAdd(reinterpret_cast<uint64_t*>(&semaphore), 1);\n }\n }\n \n block_sync::sync();\n}\n} \n\n###",940 "hip": " \nnamespace grid_sync {\n\n#define FIRST_UINT64_BIT ((uint64_t)1 << (sizeof(uint64_t) * 8 - 1))\ntemplate <typename T>\n__device__ T globalAsVolatile(volatile T& global_val) {\n return global_val;\n}\n\n\n\n\n\n\n\n\n\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK, bool PERSISTENT>\n__device__ void sync(\n int64_t& semaphore, const uint64_t& segment_size, const bool last_block) {\n \n __threadfence();\n \n block_sync::sync();\n \n if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {\n \n \n \n \n uint64_t semaphore_increment = 1;\n \n \n \n if (last_block) {\n semaphore_increment = FIRST_UINT64_BIT - (segment_size - 1);\n }\n uint64_t oldArrive =\n atomicAdd(reinterpret_cast<uint64_t*>(&semaphore), semaphore_increment);\n \n \n \n unsigned int ns = 8;\n while ((PERSISTENT || last_block) &&\n ((oldArrive ^ globalAsVolatile(semaphore)) & FIRST_UINT64_BIT) ==\n 0) {\n \n \n#if __HIP_ARCH__ >= 700\n \n __nanosleep(ns); \n if (ns < 256) {\n ns *= 2;\n }\n#endif\n }\n }\n \n block_sync::sync();\n}\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK, bool PERSISTENT>\n__device__ void sync(int64_t& semaphore, const uint64_t& segment_size) {\n sync<X_BLOCK, Y_BLOCK, Z_BLOCK, PERSISTENT>(\n semaphore, segment_size, index_utils::maskedIsLast<X_BLOCK, Y_BLOCK, Z_BLOCK>(blockIdx, gridDim));\n}\n\n\n\n\n\n\n\n\n\n\n\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK>\n__device__ void sync(\n int64_t& semaphore, const uint64_t& segment_size, const nvfuser_index_t n_entrances) {\n \n __threadfence();\n \n block_sync::sync();\n \n if (threadIdx.x == 0 && threadIdx.y == 0 && threadIdx.z == 0) {\n \n \n \n bool last_block =\n index_utils::maskedIsLast<X_BLOCK, Y_BLOCK, Z_BLOCK>(blockIdx, gridDim);\n if (last_block) {\n int64_t finished_val =\n ((int64_t)(\n index_utils::maskedSize<X_BLOCK, Y_BLOCK, Z_BLOCK>(gridDim) -\n 1)) *\n ((int64_t)n_entrances);\n unsigned int ns = 8;\n \n while (globalAsVolatile(semaphore) < finished_val) {\n#if __HIP_ARCH__ >= 700\n \n __nanosleep(ns); \n if (ns < 256) {\n ns *= 2;\n }\n#endif\n }\n } else {\n auto old = atomicAdd(reinterpret_cast<uint64_t*>(&semaphore), 1);\n }\n }\n \n block_sync::sync();\n}\n} ###"941 },942 {943 "cuda": "\nnamespace index_utils {\n\n// Utility functions\n\n// Total size of provided dimension\ntemplate <typename _dim3>\n__device__ __forceinline__ nvfuser_index_t size(const _dim3& d) {\n return (nvfuser_index_t)d.x * (nvfuser_index_t)d.y * (nvfuser_index_t)d.z;\n}\n\n// Linearized indexing of idx based on dim, if bool==false that dimension does\n// not participate\ntemplate <bool X, bool Y, bool Z, typename _dim3, typename _dim3_2>\n__device__ nvfuser_index_t maskedOffset(const _dim3& idx, const _dim3_2& dim) {\n nvfuser_index_t offset = 0;\n if (Z)\n offset += idx.z;\n if (Y)\n offset = offset * dim.y + idx.y;\n if (X)\n offset = offset * dim.x + idx.x;\n return offset;\n}\n\n// Linearized indexing of idx based on dim. All dimensions participate.\ntemplate <typename _dim3, typename _dim3_2>\n__device__ nvfuser_index_t offset(const _dim3& idx, const _dim3_2& dim) {\n nvfuser_index_t offset = idx.z;\n offset = offset * dim.y + idx.y;\n offset = offset * dim.x + idx.x;\n return offset;\n}\n\n// Masks the provided dim3, those == false get truncated to 1\ntemplate <bool X, bool Y, bool Z, typename _dim3>\n__device__ dim3 maskedDims(const _dim3& dim) {\n return dim3{\n X ? (unsigned)dim.x : 1U,\n Y ? (unsigned)dim.y : 1U,\n Z ? (unsigned)dim.z : 1U};\n}\n\n// Provides total size of dim with masking, those dims == false do not\n// participate in the size calculation\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK, typename _dim3>\n__device__ nvfuser_index_t maskedSize(const _dim3& dim) {\n return size(maskedDims<X_BLOCK, Y_BLOCK, Z_BLOCK>(dim));\n}\n\n// Checks if provided idx is zero on those dims == true\ntemplate <bool X, bool Y, bool Z, typename _dim3>\n__device__ bool maskedIsZero(const _dim3& idx) {\n bool isZero = true;\n if (X)\n isZero = isZero && idx.x == 0;\n if (Y)\n isZero = isZero && idx.y == 0;\n if (Z)\n isZero = isZero && idx.z == 0;\n return isZero;\n}\n\n// Checks if provided idx is zero on those dims == true\ntemplate <bool X, bool Y, bool Z, typename _dim3, typename _dim3_2>\n__device__ bool maskedIsLast(const _dim3& idx, const _dim3_2& dim) {\n bool isZero = true;\n if (X)\n isZero = isZero && idx.x == dim.x - 1;\n if (Y)\n isZero = isZero && idx.y == dim.y - 1;\n if (Z)\n isZero = isZero && idx.z == dim.z - 1;\n return isZero;\n}\n\n} // namespace index_utils\n\n\n###",944 "hip": " // !!! This is a file automatically generated by hipify!!!\nnamespace index_utils {\n\n// Utility functions\n\n// Total size of provided dimension\ntemplate <typename _dim3>\n__device__ __forceinline__ nvfuser_index_t size(const _dim3& d) {\n return (nvfuser_index_t)d.x * (nvfuser_index_t)d.y * (nvfuser_index_t)d.z;\n}\n\n// Linearized indexing of idx based on dim, if bool==false that dimension does\n// not participate\ntemplate <bool X, bool Y, bool Z, typename _dim3, typename _dim3_2>\n__device__ nvfuser_index_t maskedOffset(const _dim3& idx, const _dim3_2& dim) {\n nvfuser_index_t offset = 0;\n if (Z)\n offset += idx.z;\n if (Y)\n offset = offset * dim.y + idx.y;\n if (X)\n offset = offset * dim.x + idx.x;\n return offset;\n}\n\n// Linearized indexing of idx based on dim. All dimensions participate.\ntemplate <typename _dim3, typename _dim3_2>\n__device__ nvfuser_index_t offset(const _dim3& idx, const _dim3_2& dim) {\n nvfuser_index_t offset = idx.z;\n offset = offset * dim.y + idx.y;\n offset = offset * dim.x + idx.x;\n return offset;\n}\n\n// Masks the provided dim3, those == false get truncated to 1\ntemplate <bool X, bool Y, bool Z, typename _dim3>\n__device__ dim3 maskedDims(const _dim3& dim) {\n return dim3{\n X ? (unsigned)dim.x : 1U,\n Y ? (unsigned)dim.y : 1U,\n Z ? (unsigned)dim.z : 1U};\n}\n\n// Provides total size of dim with masking, those dims == false do not\n// participate in the size calculation\ntemplate <bool X_BLOCK, bool Y_BLOCK, bool Z_BLOCK, typename _dim3>\n__device__ nvfuser_index_t maskedSize(const _dim3& dim) {\n return size(maskedDims<X_BLOCK, Y_BLOCK, Z_BLOCK>(dim));\n}\n\n// Checks if provided idx is zero on those dims == true\ntemplate <bool X, bool Y, bool Z, typename _dim3>\n__device__ bool maskedIsZero(const _dim3& idx) {\n bool isZero = true;\n if (X)\n isZero = isZero && idx.x == 0;\n if (Y)\n isZero = isZero && idx.y == 0;\n if (Z)\n isZero = isZero && idx.z == 0;\n return isZero;\n}\n\n// Checks if provided idx is zero on those dims == true\ntemplate <bool X, bool Y, bool Z, typename _dim3, typename _dim3_2>\n__device__ bool maskedIsLast(const _dim3& idx, const _dim3_2& dim) {\n bool isZero = true;\n if (X)\n isZero = isZero && idx.x == dim.x - 1;\n if (Y)\n isZero = isZero && idx.y == dim.y - 1;\n if (Z)\n isZero = isZero && idx.z == dim.z - 1;\n return isZero;\n}\n\n} // namespace index_utils\n###"945 },946 {947 "cuda": "\n__device__ unsigned int mulhilo32(\n unsigned int a,\n unsigned int b,\n unsigned int* result_high) {\n *result_high = __umulhi(a, b);\n return a * b;\n}\n\n__device__ uint4 single_round(uint4 ctr, uint2 key) {\n constexpr unsigned long kPhiloxSA = 0xD2511F53;\n constexpr unsigned long kPhiloxSB = 0xCD9E8D57;\n unsigned int hi0;\n unsigned int hi1;\n unsigned int lo0 = mulhilo32(kPhiloxSA, ctr.x, &hi0);\n unsigned int lo1 = mulhilo32(kPhiloxSB, ctr.z, &hi1);\n uint4 ret = {hi1 ^ ctr.y ^ key.x, lo1, hi0 ^ ctr.w ^ key.y, lo0};\n return ret;\n}\n\n__device__ uint4 philox(\n unsigned long long seed,\n unsigned long long subsequence,\n unsigned long long offset) {\n constexpr unsigned long kPhilox10A = 0x9E3779B9;\n constexpr unsigned long kPhilox10B = 0xBB67AE85;\n uint2 key = {};\n key.x = (unsigned int)seed;\n key.y = (unsigned int)(seed >> 32);\n uint4 counter = make_uint4(0, 0, 0, 0);\n counter.x = (unsigned int)(offset);\n counter.y = (unsigned int)(offset >> 32);\n counter.z = (unsigned int)(subsequence);\n counter.w = (unsigned int)(subsequence >> 32);\n\n uint4 output = {};\n uint2 key_ = key;\n uint4 counter_ = counter;\n for (int i = 0; i < 9; i++) {\n counter_ = single_round(counter_, key_);\n key_.x += (kPhilox10A);\n key_.y += (kPhilox10B);\n }\n output = single_round(counter_, key_);\n return output;\n}\n\n__device__ float uniformf(unsigned int x) {\n constexpr float kRanInvM32 = 2.3283064e-10f; // Inverse of 2^32.\n float result = x * kRanInvM32;\n return result == 1 ? 0.0f : result;\n}\n\n__device__ double uniform(unsigned int x, unsigned int y) {\n constexpr double kRan2Pow53Inv = 1.1102230246251565e-16;\n const unsigned long long z =\n (unsigned long long)x ^ ((unsigned long long)y << (53 - 32));\n double result = z * kRan2Pow53Inv + (kRan2Pow53Inv / 2.0);\n return result == 1 ? 0.0 : result;\n}\n\n__device__ double rng_uniform(const uint4& rng_result, int rng_component) {\n return uniform(\n (&rng_result.x)[rng_component * 2],\n (&rng_result.x)[rng_component * 2 + 1]);\n}\n\n__device__ float rng_uniformf(const uint4& rng_result, int rng_component) {\n return uniformf((&rng_result.x)[rng_component]);\n}\n\n__device__ double rng_uniform_range(\n const uint4& rng_result,\n int rng_component,\n double from,\n double to) {\n auto range = to - from;\n auto uniform01 = rng_uniform(rng_result, rng_component);\n return from + range * uniform01;\n}\n\n__device__ float rng_uniform_rangef(\n const uint4& rng_result,\n int rng_component,\n float from,\n float to) {\n auto range = to - from;\n auto uniform01 = rng_uniformf(rng_result, rng_component);\n return from + range * uniform01;\n}\n\n\n###",948 "hip": " \n__device__ unsigned int mulhilo32(\n unsigned int a, unsigned int b, unsigned int* result_high) {\n *result_high = __umulhi(a, b);\n return a * b;\n}\n__device__ uint4 single_round(uint4 ctr, uint2 key) {\n constexpr unsigned long kPhiloxSA = 0xD2511F53;\n constexpr unsigned long kPhiloxSB = 0xCD9E8D57;\n unsigned int hi0;\n unsigned int hi1;\n unsigned int lo0 = mulhilo32(kPhiloxSA, ctr.x, &hi0);\n unsigned int lo1 = mulhilo32(kPhiloxSB, ctr.z, &hi1);\n uint4 ret = {hi1 ^ ctr.y ^ key.x, lo1, hi0 ^ ctr.w ^ key.y, lo0};\n return ret;\n}\n__device__ uint4 philox(\n unsigned long long seed, unsigned long long subsequence, unsigned long long offset) {\n constexpr unsigned long kPhilox10A = 0x9E3779B9;\n constexpr unsigned long kPhilox10B = 0xBB67AE85;\n uint2 key = {};\n key.x = (unsigned int)seed;\n key.y = (unsigned int)(seed >> 32);\n uint4 counter = make_uint4(0, 0, 0, 0);\n counter.x = (unsigned int)(offset);\n counter.y = (unsigned int)(offset >> 32);\n counter.z = (unsigned int)(subsequence);\n counter.w = (unsigned int)(subsequence >> 32);\n uint4 output = {};\n uint2 key_ = key;\n uint4 counter_ = counter;\n for (int i = 0; i < 9; i++) {\n counter_ = single_round(counter_, key_);\n key_.x += (kPhilox10A);\n key_.y += (kPhilox10B);\n }\n output = single_round(counter_, key_);\n return output;\n}\n__device__ float uniformf(unsigned int x) {\n constexpr float kRanInvM32 = 2.3283064e-10f; \n float result = x * kRanInvM32;\n return result == 1 ? 0.0f : result;\n}\n__device__ double uniform(unsigned int x, unsigned int y) {\n constexpr double kRan2Pow53Inv = 1.1102230246251565e-16;\n const unsigned long long z =\n (unsigned long long)x ^ ((unsigned long long)y << (53 - 32));\n double result = z * kRan2Pow53Inv + (kRan2Pow53Inv / 2.0);\n return result == 1 ? 0.0 : result;\n}\n__device__ double rng_uniform(const uint4& rng_result, int rng_component) {\n return uniform(\n (&rng_result.x)[rng_component * 2], (&rng_result.x)[rng_component * 2 + 1]);\n}\n__device__ float rng_uniformf(const uint4& rng_result, int rng_component) {\n return uniformf((&rng_result.x)[rng_component]);\n}\n__device__ double rng_uniform_range(\n const uint4& rng_result, int rng_component, double from, double to) {\n auto range = to - from;\n auto uniform01 = rng_uniform(rng_result, rng_component);\n return from + range * uniform01;\n}\n__device__ float rng_uniform_rangef(\n const uint4& rng_result, int rng_component, float from, float to) {\n auto range = to - from;\n auto uniform01 = rng_uniformf(rng_result, rng_component);\n return from + range * uniform01;\n}###"949 },950 {951 "cuda": "\n// Utility macro for this file\n#define DEVICE_INLINE __device__ inline\n\n// Utility class for 2D swizzle:\ntemplate <typename index_t>\nstruct IndexGeneric {\n const index_t x = 0, y = 0;\n DEVICE_INLINE IndexGeneric(index_t x_, index_t y_) : x(x_), y(y_) {}\n};\n\n// Default type for integration\nusing Index2D = IndexGeneric<nvfuser_index_t>;\n\n// Small type for unit computation\nusing Index2DInt = IndexGeneric<int>;\n\n// ------------------------------------------------------------\n// Swizzle Definitions\n// for each swizzle name:\n// un(Swizzle Name) e.g. unZShape is the inverse of ZShape,\n// (unswizzle is needed for inlining and is currently not actively used.)\n// ------------------------------------------------------------\n\n// Unit Z swizzle:\n// Alternate directions of Y dimension:\n// 1 2 3 1 2 3\n// 4 5 6 => 6 5 4\n// 7 8 9 7 8 9\nDEVICE_INLINE Index2D ZShape(Index2D in, Index2D unit_dim) {\n return Index2D(in.x, in.x % 2 == 0 ? in.y : (unit_dim.y - in.y - 1));\n}\n\n// ZShape is inverse of itself\nDEVICE_INLINE Index2D unZShape(Index2D in, Index2D unit_dim) {\n return ZShape(in, unit_dim);\n}\n\n// Block cyclic Xor swizzle: (bank conflict removal)\n// Apply cyclic Xor within blocks:\n// Example: cyclic Xor\n// 1 2 3 4 1 2 3 4\n// 5 6 7 8 6 5 8 7\n// 9 10 11 12 => 11 12 9 10\n// 13 14 15 16 16 15 14 13\n// Note:\nDEVICE_INLINE Index2D Xor(Index2D in, Index2DInt unit_dim) {\n // Need to validate in swizzle configuration:\n // unit_dim.x == unit_dim.y\n return Index2D(in.x, (in.y ^ in.x));\n}\n\n// Inverse of Xor is itself\nDEVICE_INLINE Index2D unXor(Index2D in, Index2DInt unit_dim) {\n return Xor(in, unit_dim);\n}\n\n// Scatter swizzle:\n// Corresponds to the data layout out of ldmatrix intrinsic.\n// supported dimensions are : 8x4, 16x4, 32x4\ntemplate <int row_size>\nDEVICE_INLINE Index2D Scatter(Index2D in) {\n static_assert(row_size == 8 || row_size == 16 || row_size == 32);\n return Index2D((in.y * row_size + in.x) / 4, in.x % 4);\n}\n\ntemplate <int row_size>\nDEVICE_INLINE Index2D unScatter(Index2D in) {\n static_assert(row_size == 8 || row_size == 16 || row_size == 32);\n return Index2D(in.y + (in.x % (row_size / 4)) * 4, in.x / (row_size / 4));\n}\n\n#undef DEVICE_INLINE\n\n\n###",952 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Utility macro for this file\n#define DEVICE_INLINE __device__ inline\n\n// Utility class for 2D swizzle:\ntemplate <typename index_t>\nstruct IndexGeneric {\n const index_t x = 0, y = 0;\n DEVICE_INLINE IndexGeneric(index_t x_, index_t y_) : x(x_), y(y_) {}\n};\n\n// Default type for integration\nusing Index2D = IndexGeneric<nvfuser_index_t>;\n\n// Small type for unit computation\nusing Index2DInt = IndexGeneric<int>;\n\n// ------------------------------------------------------------\n// Swizzle Definitions\n// for each swizzle name:\n// un(Swizzle Name) e.g. unZShape is the inverse of ZShape,\n// (unswizzle is needed for inlining and is currently not actively used.)\n// ------------------------------------------------------------\n\n// Unit Z swizzle:\n// Alternate directions of Y dimension:\n// 1 2 3 1 2 3\n// 4 5 6 => 6 5 4\n// 7 8 9 7 8 9\nDEVICE_INLINE Index2D ZShape(Index2D in, Index2D unit_dim) {\n return Index2D(in.x, in.x % 2 == 0 ? in.y : (unit_dim.y - in.y - 1));\n}\n\n// ZShape is inverse of itself\nDEVICE_INLINE Index2D unZShape(Index2D in, Index2D unit_dim) {\n return ZShape(in, unit_dim);\n}\n\n// Block cyclic Xor swizzle: (bank conflict removal)\n// Apply cyclic Xor within blocks:\n// Example: cyclic Xor\n// 1 2 3 4 1 2 3 4\n// 5 6 7 8 6 5 8 7\n// 9 10 11 12 => 11 12 9 10\n// 13 14 15 16 16 15 14 13\n// Note:\nDEVICE_INLINE Index2D Xor(Index2D in, Index2DInt unit_dim) {\n // Need to validate in swizzle configuration:\n // unit_dim.x == unit_dim.y\n return Index2D(in.x, (in.y ^ in.x));\n}\n\n// Inverse of Xor is itself\nDEVICE_INLINE Index2D unXor(Index2D in, Index2DInt unit_dim) {\n return Xor(in, unit_dim);\n}\n\n// Scatter swizzle:\n// Corresponds to the data layout out of ldmatrix intrinsic.\n// supported dimensions are : 8x4, 16x4, 32x4\ntemplate <int row_size>\nDEVICE_INLINE Index2D Scatter(Index2D in) {\n static_assert(row_size == 8 || row_size == 16 || row_size == 32);\n return Index2D((in.y * row_size + in.x) / 4, in.x % 4);\n}\n\ntemplate <int row_size>\nDEVICE_INLINE Index2D unScatter(Index2D in) {\n static_assert(row_size == 8 || row_size == 16 || row_size == 32);\n return Index2D(in.y + (in.x % (row_size / 4)) * 4, in.x / (row_size / 4));\n}\n\n#undef DEVICE_INLINE\n###"953 },954 {955 "cuda": "\ntemplate <typename T, int N>\nstruct Tensor {\n __device__ T& operator[](nvfuser_index_t ind) {\n return data[ind];\n };\n\n T* data;\n nvfuser_index_t size[N];\n nvfuser_index_t stride[N];\n};\n\n// Specialization for 0-dim case as it does not need size and stride arrays.\n// They will be an error as well since zero-length arrays are not allowed.\ntemplate <typename T>\nstruct Tensor<T, 0> {\n __device__ T& operator[](nvfuser_index_t) {\n return *data;\n };\n\n T* data;\n};\n\n// Specialization for 0-dim case that's easy to pass in a CPU based tensor.\ntemplate <typename T>\nstruct CpuScalarTensor {\n __device__ T& operator[](int) {\n return data;\n };\n\n T data;\n};\n\n\n###",956 "hip": " // !!! This is a file automatically generated by hipify!!!\ntemplate <typename T, int N>\nstruct Tensor {\n __device__ T& operator[](nvfuser_index_t ind) {\n return data[ind];\n };\n\n T* data;\n nvfuser_index_t size[N];\n nvfuser_index_t stride[N];\n};\n\n// Specialization for 0-dim case as it does not need size and stride arrays.\n// They will be an error as well since zero-length arrays are not allowed.\ntemplate <typename T>\nstruct Tensor<T, 0> {\n __device__ T& operator[](nvfuser_index_t) {\n return *data;\n };\n\n T* data;\n};\n\n// Specialization for 0-dim case that's easy to pass in a CPU based tensor.\ntemplate <typename T>\nstruct CpuScalarTensor {\n __device__ T& operator[](int) {\n return data;\n };\n\n T data;\n};\n###"957 },958 {959 "cuda": "\n// Type trait utils\ntemplate <typename Type, bool is_volatile>\nstruct MaybeVolatile;\n\ntemplate <typename Type>\nstruct MaybeVolatile<Type, true> {\n using type = volatile Type;\n};\n\ntemplate <typename Type>\nstruct MaybeVolatile<Type, false> {\n using type = Type;\n};\n\ntemplate <typename... Types>\nstruct TypeList {};\n\ntemplate <int idx, typename T, typename... Types>\nstruct TypeSelector {\n using type = typename TypeSelector<idx - 1, Types...>::type;\n};\n\ntemplate <typename T, typename... Types>\nstruct TypeSelector<0, T, Types...> {\n using type = T;\n};\n\ntemplate <typename T0, typename T1>\nstruct IsSameType {\n static constexpr bool value = false;\n};\n\ntemplate <typename T0>\nstruct IsSameType<T0, T0> {\n static constexpr bool value = true;\n};\n\ntemplate <typename T>\nstruct IsPointerType {\n static constexpr bool value = false;\n};\n\ntemplate <typename T>\nstruct IsPointerType<T*> {\n static constexpr bool value = true;\n};\n\n\n###",960 "hip": " // !!! This is a file automatically generated by hipify!!!\n// Type trait utils\ntemplate <typename Type, bool is_volatile>\nstruct MaybeVolatile;\n\ntemplate <typename Type>\nstruct MaybeVolatile<Type, true> {\n using type = volatile Type;\n};\n\ntemplate <typename Type>\nstruct MaybeVolatile<Type, false> {\n using type = Type;\n};\n\ntemplate <typename... Types>\nstruct TypeList {};\n\ntemplate <int idx, typename T, typename... Types>\nstruct TypeSelector {\n using type = typename TypeSelector<idx - 1, Types...>::type;\n};\n\ntemplate <typename T, typename... Types>\nstruct TypeSelector<0, T, Types...> {\n using type = T;\n};\n\ntemplate <typename T0, typename T1>\nstruct IsSameType {\n static constexpr bool value = false;\n};\n\ntemplate <typename T0>\nstruct IsSameType<T0, T0> {\n static constexpr bool value = true;\n};\n\ntemplate <typename T>\nstruct IsPointerType {\n static constexpr bool value = false;\n};\n\ntemplate <typename T>\nstruct IsPointerType<T*> {\n static constexpr bool value = true;\n};\n###"961 },962 {963 "cuda": "\nnamespace warp {\n\ntemplate <\n bool SINGLE_WARP,\n typename T,\n typename Func,\n typename _dim3ti,\n typename _dim3bd>\n__device__ void warpReduceTIDX(\n T& out,\n const T& inp_val,\n Func reduction_op,\n const _dim3ti& thread_idx,\n const _dim3bd& block_dim,\n T* shared_mem,\n bool read_write_pred,\n T init_val) {\n constexpr int WARP_SIZE = 32;\n\n // Assume input padded to multiples of a warp\n T reduce_val = init_val;\n\n // Do warp reduction\n if (read_write_pred) {\n reduce_val = inp_val;\n }\n\n // Reduce within each warp\n for (int i = 16; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor_sync(0xffffffff, reduce_val, i, WARP_SIZE));\n }\n\n // Reduce across warp if needed\n // Load value to shared mem\n if (!SINGLE_WARP) {\n unsigned int warp_idx = thread_idx.x / WARP_SIZE;\n unsigned int lane_idx = thread_idx.x % WARP_SIZE;\n unsigned int reduce_group_id = thread_idx.z * block_dim.y + thread_idx.y;\n bool is_warp_head = lane_idx == 0;\n unsigned int reduction_size = block_dim.x;\n unsigned int num_of_warps = reduction_size / WARP_SIZE;\n unsigned int smem_offset = reduce_group_id * num_of_warps;\n\n block_sync::sync();\n\n if (is_warp_head) {\n shared_mem[smem_offset + warp_idx] = reduce_val;\n }\n\n block_sync::sync();\n\n if (warp_idx == 0) {\n // This assumes num_of_warps will be < 32, meaning < 1024 threads.\n // Should be true for long enough.\n assert(num_of_warps <= 32);\n\n reduce_val = lane_idx < num_of_warps ? shared_mem[smem_offset + lane_idx]\n : init_val;\n\n // Reduce within warp 0\n for (int i = 16; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor_sync(0xffffffff, reduce_val, i, 32));\n }\n }\n\n if (is_warp_head) {\n reduction_op(out, reduce_val);\n }\n } else {\n reduction_op(out, reduce_val);\n }\n}\n\n} // namespace warp\n\n\n###",964 "hip": " // !!! This is a file automatically generated by hipify!!!\nnamespace warp {\n\ntemplate <\n bool SINGLE_WARP,\n typename T,\n typename Func,\n typename _dim3ti,\n typename _dim3bd>\n__device__ void warpReduceTIDX(\n T& out,\n const T& inp_val,\n Func reduction_op,\n const _dim3ti& thread_idx,\n const _dim3bd& block_dim,\n T* shared_mem,\n bool read_write_pred,\n T init_val) {\n constexpr int WARP_SIZE = 32;\n\n // Assume input padded to multiples of a warp\n T reduce_val = init_val;\n\n // Do warp reduction\n if (read_write_pred) {\n reduce_val = inp_val;\n }\n\n // Reduce within each warp\n for (int i = 16; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor_sync(0xffffffff, reduce_val, i, WARP_SIZE));\n }\n\n // Reduce across warp if needed\n // Load value to shared mem\n if (!SINGLE_WARP) {\n unsigned int warp_idx = thread_idx.x / WARP_SIZE;\n unsigned int lane_idx = thread_idx.x % WARP_SIZE;\n unsigned int reduce_group_id = thread_idx.z * block_dim.y + thread_idx.y;\n bool is_warp_head = lane_idx == 0;\n unsigned int reduction_size = block_dim.x;\n unsigned int num_of_warps = reduction_size / WARP_SIZE;\n unsigned int smem_offset = reduce_group_id * num_of_warps;\n\n block_sync::sync();\n\n if (is_warp_head) {\n shared_mem[smem_offset + warp_idx] = reduce_val;\n }\n\n block_sync::sync();\n\n if (warp_idx == 0) {\n // This assumes num_of_warps will be < 32, meaning < 1024 threads.\n // Should be true for long enough.\n assert(num_of_warps <= 32);\n\n reduce_val = lane_idx < num_of_warps ? shared_mem[smem_offset + lane_idx]\n : init_val;\n\n // Reduce within warp 0\n for (int i = 16; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor_sync(0xffffffff, reduce_val, i, 32));\n }\n }\n\n if (is_warp_head) {\n reduction_op(out, reduce_val);\n }\n } else {\n reduction_op(out, reduce_val);\n }\n}\n\n} // namespace warp\n###"965 },966 {967 "cuda": "\nnamespace warp {\n\ntemplate <\n bool SINGLE_WARP,\n typename T,\n typename Func,\n typename _dim3ti,\n typename _dim3bd>\n__device__ void warpReduceTIDX(\n T& out,\n const T& inp_val,\n Func reduction_op,\n const _dim3ti& thread_idx,\n const _dim3bd& block_dim,\n T* shared_mem,\n bool read_write_pred,\n T init_val) {\n constexpr int WARP_SIZE = warpSize;\n\n // Assume input padded to multiples of a warp\n T reduce_val = init_val;\n\n // Do warp reduction\n if (read_write_pred) {\n reduce_val = inp_val;\n }\n\n // Reduce within each warp\n for (int i = WARP_SIZE/2; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor(reduce_val, i, WARP_SIZE));\n }\n\n // Reduce across warp if needed\n // Load value to shared mem\n if (!SINGLE_WARP) {\n unsigned int warp_idx = thread_idx.x / WARP_SIZE;\n unsigned int lane_idx = thread_idx.x % WARP_SIZE;\n unsigned int reduce_group_id = thread_idx.z * block_dim.y + thread_idx.y;\n bool is_warp_head = lane_idx == 0;\n unsigned int reduction_size = block_dim.x;\n unsigned int num_of_warps = reduction_size / WARP_SIZE;\n unsigned int smem_offset = reduce_group_id * num_of_warps;\n\n block_sync::sync();\n\n if (read_write_pred && is_warp_head) {\n shared_mem[smem_offset + warp_idx] = reduce_val;\n }\n\n block_sync::sync();\n\n if (warp_idx == 0) {\n // This assumes num_of_warps will be < 32, meaning < 1024 threads.\n // Should be true for long enough.\n assert(num_of_warps <= 32);\n\n reduce_val = lane_idx < num_of_warps ? shared_mem[smem_offset + lane_idx]\n : init_val;\n\n // Reduce within warp 0\n for (int i = WARP_SIZE/2; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor(reduce_val, i, WARP_SIZE));\n }\n }\n\n if (is_warp_head) {\n reduction_op(out, reduce_val);\n }\n } else {\n reduction_op(out, reduce_val);\n }\n}\n\n} // namespace warp\n\n\n###",968 "hip": " // !!! This is a file automatically generated by hipify!!!\nnamespace warp {\n\ntemplate <\n bool SINGLE_WARP,\n typename T,\n typename Func,\n typename _dim3ti,\n typename _dim3bd>\n__device__ void warpReduceTIDX(\n T& out,\n const T& inp_val,\n Func reduction_op,\n const _dim3ti& thread_idx,\n const _dim3bd& block_dim,\n T* shared_mem,\n bool read_write_pred,\n T init_val) {\n constexpr int WARP_SIZE = warpSize;\n\n // Assume input padded to multiples of a warp\n T reduce_val = init_val;\n\n // Do warp reduction\n if (read_write_pred) {\n reduce_val = inp_val;\n }\n\n // Reduce within each warp\n for (int i = WARP_SIZE/2; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor(reduce_val, i, WARP_SIZE));\n }\n\n // Reduce across warp if needed\n // Load value to shared mem\n if (!SINGLE_WARP) {\n unsigned int warp_idx = thread_idx.x / WARP_SIZE;\n unsigned int lane_idx = thread_idx.x % WARP_SIZE;\n unsigned int reduce_group_id = thread_idx.z * block_dim.y + thread_idx.y;\n bool is_warp_head = lane_idx == 0;\n unsigned int reduction_size = block_dim.x;\n unsigned int num_of_warps = reduction_size / WARP_SIZE;\n unsigned int smem_offset = reduce_group_id * num_of_warps;\n\n block_sync::sync();\n\n if (read_write_pred && is_warp_head) {\n shared_mem[smem_offset + warp_idx] = reduce_val;\n }\n\n block_sync::sync();\n\n if (warp_idx == 0) {\n // This assumes num_of_warps will be < 32, meaning < 1024 threads.\n // Should be true for long enough.\n assert(num_of_warps <= 32);\n\n reduce_val = lane_idx < num_of_warps ? shared_mem[smem_offset + lane_idx]\n : init_val;\n\n // Reduce within warp 0\n for (int i = WARP_SIZE/2; i >= 1; i /= 2) {\n reduction_op(\n reduce_val, __shfl_xor(reduce_val, i, WARP_SIZE));\n }\n }\n\n if (is_warp_head) {\n reduction_op(out, reduce_val);\n }\n } else {\n reduction_op(out, reduce_val);\n }\n}\n\n} // namespace warp\n###"969 },970 {971 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_CUDA char bessel_j1_name[] = \"bessel_j1_forward\";\n\nvoid bessel_j1_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j1_cuda\", [&]() {\n jitted_gpu_kernel<bessel_j1_name, scalar_t, scalar_t, 1>(iterator, bessel_j1_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j1_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_j1_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(special_bessel_j1_stub, &bessel_j1_kernel_cuda);\n} // namespace at::native\n\n\n###",972 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\nnamespace {\nCONSTEXPR_EXCEPT_WIN_HIP char bessel_j1_name[] = \"bessel_j1_forward\";\n\nvoid bessel_j1_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j1_hip\", [&]() {\n jitted_gpu_kernel<bessel_j1_name, scalar_t, scalar_t, 1>(iterator, bessel_j1_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_j1_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_j1_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(special_bessel_j1_stub, &bessel_j1_kernel_hip);\n} // namespace at::native\n###"973 },974 {975 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char bessel_y0_name[] = \"bessel_y0_forward\";\n\n void bessel_y0_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y0_cuda\", [&]() {\n jitted_gpu_kernel<bessel_y0_name, scalar_t, scalar_t, 1>(iterator, bessel_y0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y0_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_y0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_bessel_y0_stub, &bessel_y0_kernel_cuda);\n} // namespace at::native\n\n\n###",976 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char bessel_y0_name[] = \"bessel_y0_forward\";\n\n void bessel_y0_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y0_hip\", [&]() {\n jitted_gpu_kernel<bessel_y0_name, scalar_t, scalar_t, 1>(iterator, bessel_y0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y0_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_y0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_bessel_y0_stub, &bessel_y0_kernel_hip);\n} // namespace at::native\n###"977 },978 {979 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char bessel_y1_name[] = \"bessel_y1_forward\";\n\n void bessel_y1_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y1_cuda\", [&]() {\n jitted_gpu_kernel<bessel_y1_name, scalar_t, scalar_t, 1>(iterator, bessel_y1_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y1_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_y1_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_bessel_y1_stub, &bessel_y1_kernel_cuda);\n} // namespace at::native\n\n\n###",980 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char bessel_y1_name[] = \"bessel_y1_forward\";\n\n void bessel_y1_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y1_hip\", [&]() {\n jitted_gpu_kernel<bessel_y1_name, scalar_t, scalar_t, 1>(iterator, bessel_y1_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"bessel_y1_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return bessel_y1_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_bessel_y1_stub, &bessel_y1_kernel_hip);\n} // namespace at::native\n###"981 },982 {983 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\ntemplate<typename scalar_t>\nstruct BitwiseAndFunctor {\n __device__ __forceinline__ scalar_t operator()(scalar_t a, scalar_t b) const {\n return a & b;\n }\n};\n\ntemplate<>\nstruct BitwiseAndFunctor<bool> {\n __device__ __forceinline__ bool operator()(bool a, bool b) const {\n return a && b;\n }\n};\n\nvoid bitwise_and_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES_AND(kBool, iter.dtype(), \"bitwise_and_cuda\", [&]() {\n BitwiseAndFunctor<scalar_t> f;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(iter, f);\n });\n}\n\ntemplate<typename scalar_t>\nstruct BitwiseOrFunctor {\n __device__ __forceinline__ scalar_t operator()(scalar_t a, scalar_t b) const {\n return a | b;\n }\n};\n\ntemplate<>\nstruct BitwiseOrFunctor<bool> {\n __device__ __forceinline__ bool operator()(bool a, bool b) const {\n return a || b;\n }\n};\n\nvoid bitwise_or_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES_AND(kBool, iter.dtype(), \"bitwise_or_cuda\", [&]() {\n BitwiseOrFunctor<scalar_t> f;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(iter, f);\n });\n}\n\ntemplate<typename scalar_t>\nstruct BitwiseXorFunctor {\n __device__ __forceinline__ scalar_t operator()(scalar_t a, scalar_t b) const {\n return a ^ b;\n }\n};\n\ntemplate<>\nstruct BitwiseXorFunctor<bool> {\n __device__ __forceinline__ bool operator()(bool a, bool b) const {\n return a != b;\n }\n};\n\nvoid bitwise_xor_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES_AND(kBool, iter.dtype(), \"bitwise_xor_cuda\", [&]() {\n BitwiseXorFunctor<scalar_t> f;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(iter, f);\n });\n}\n\nREGISTER_DISPATCH(bitwise_and_stub, &bitwise_and_kernel_cuda);\nREGISTER_DISPATCH(bitwise_or_stub, &bitwise_or_kernel_cuda);\nREGISTER_DISPATCH(bitwise_xor_stub, &bitwise_xor_kernel_cuda);\n\n\n} // namespace at::native\n\n\n###",984 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\ntemplate<typename scalar_t>\nstruct BitwiseAndFunctor {\n __device__ __forceinline__ scalar_t operator()(scalar_t a, scalar_t b) const {\n return a & b;\n }\n};\n\ntemplate<>\nstruct BitwiseAndFunctor<bool> {\n __device__ __forceinline__ bool operator()(bool a, bool b) const {\n return a && b;\n }\n};\n\nvoid bitwise_and_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES_AND(kBool, iter.dtype(), \"bitwise_and_hip\", [&]() {\n BitwiseAndFunctor<scalar_t> f;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(iter, f);\n });\n}\n\ntemplate<typename scalar_t>\nstruct BitwiseOrFunctor {\n __device__ __forceinline__ scalar_t operator()(scalar_t a, scalar_t b) const {\n return a | b;\n }\n};\n\ntemplate<>\nstruct BitwiseOrFunctor<bool> {\n __device__ __forceinline__ bool operator()(bool a, bool b) const {\n return a || b;\n }\n};\n\nvoid bitwise_or_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES_AND(kBool, iter.dtype(), \"bitwise_or_hip\", [&]() {\n BitwiseOrFunctor<scalar_t> f;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(iter, f);\n });\n}\n\ntemplate<typename scalar_t>\nstruct BitwiseXorFunctor {\n __device__ __forceinline__ scalar_t operator()(scalar_t a, scalar_t b) const {\n return a ^ b;\n }\n};\n\ntemplate<>\nstruct BitwiseXorFunctor<bool> {\n __device__ __forceinline__ bool operator()(bool a, bool b) const {\n return a != b;\n }\n};\n\nvoid bitwise_xor_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES_AND(kBool, iter.dtype(), \"bitwise_xor_hip\", [&]() {\n BitwiseXorFunctor<scalar_t> f;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(iter, f);\n });\n}\n\nREGISTER_DISPATCH(bitwise_and_stub, &bitwise_and_kernel_hip);\nREGISTER_DISPATCH(bitwise_or_stub, &bitwise_or_kernel_hip);\nREGISTER_DISPATCH(bitwise_xor_stub, &bitwise_xor_kernel_hip);\n\n\n} // namespace at::native\n###"985 },986 {987 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/BinaryInternal.h>\n#include <c10/cuda/CUDAGuard.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/cuda/BinaryInternal.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <type_traits>\nnamespace at::native {\nnamespace binary_internal {\nvoid div_floor_kernel_cuda(TensorIteratorBase& iter) {\n \n const auto dtype = iter.common_dtype();\n if (dtype == kByte) {\n \n \n \n return div_trunc_kernel_cuda(iter);\n } else if (isIntegralType(dtype, false)) {\n AT_DISPATCH_INTEGRAL_TYPES(dtype, \"div_floor_cuda\", [&]() {\n gpu_kernel_with_scalars(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return div_floor_integer(a, b);\n });\n });\n } else if (iter.is_cpu_scalar(2)) {\n \n \n \n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_floor_cuda\", [&]() {\n using accscalar_t = at::acc_type<scalar_t, true>;\n auto b = iter.scalar_value<accscalar_t>(2);\n if (C10_UNLIKELY(b == 0)) {\n return div_true_kernel_cuda(iter);\n }\n auto inv_b = accscalar_t(1.0) / b;\n iter.remove_operand(2);\n gpu_kernel(iter, [b, inv_b] GPU_LAMBDA(scalar_t a) -> scalar_t {\n auto mod = std::fmod(a, b);\n auto div = (a - mod) * inv_b;\n if ((mod != 0) && (b < 0) != (mod < 0)) {\n div -= scalar_t(1);\n }\n scalar_t floordiv;\n if (div != 0) {\n floordiv = std::floor(div);\n if (div - floordiv > scalar_t(0.5)) {\n floordiv += scalar_t(1.0);\n }\n } else {\n floordiv = c10::cuda::compat::copysign(scalar_t(0), a * inv_b);\n }\n return floordiv;\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_floor_cuda\", [&]() {\n gpu_kernel_with_scalars(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return div_floor_floating(a, b);\n });\n });\n }\n}\n} \nREGISTER_DISPATCH(div_floor_stub, &binary_internal::div_floor_kernel_cuda);\n} \n\n###",988 "hip": " \n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\BinaryInternal.h>\n#include <c10/hip/HIPGuard.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/hip\\BinaryInternal.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <type_traits>\nnamespace at::native {\nnamespace binary_internal {\nvoid div_floor_kernel_hip(TensorIteratorBase& iter) {\n \n const auto dtype = iter.common_dtype();\n if (dtype == kByte) {\n \n \n \n return div_trunc_kernel_hip(iter);\n } else if (isIntegralType(dtype, false)) {\n AT_DISPATCH_INTEGRAL_TYPES(dtype, \"div_floor_hip\", [&]() {\n gpu_kernel_with_scalars(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return div_floor_integer(a, b);\n });\n });\n } else if (iter.is_cpu_scalar(2)) {\n \n \n \n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_floor_hip\", [&]() {\n using accscalar_t = at::acc_type<scalar_t, true>;\n auto b = iter.scalar_value<accscalar_t>(2);\n if (C10_UNLIKELY(b == 0)) {\n return div_true_kernel_hip(iter);\n }\n auto inv_b = accscalar_t(1.0) / b;\n iter.remove_operand(2);\n gpu_kernel(iter, [b, inv_b] GPU_LAMBDA(scalar_t a) -> scalar_t {\n auto mod = ::fmod(a, b);\n auto div = (a - mod) * inv_b;\n if ((mod != 0) && (b < 0) != (mod < 0)) {\n div -= scalar_t(1);\n }\n scalar_t floordiv;\n if (div != 0) {\n floordiv = ::floor(div);\n if (div - floordiv > scalar_t(0.5)) {\n floordiv += scalar_t(1.0);\n }\n } else {\n floordiv = c10::hip::compat::copysign(scalar_t(0), a * inv_b);\n }\n return floordiv;\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_floor_hip\", [&]() {\n gpu_kernel_with_scalars(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return div_floor_floating(a, b);\n });\n });\n }\n}\n} \nREGISTER_DISPATCH(div_floor_stub, &binary_internal::div_floor_kernel_hip);\n} ###"989 },990 {991 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <c10/cuda/CUDAGuard.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/cuda/BinaryInternal.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\n#include <type_traits>\n\nnamespace at::native {\nnamespace binary_internal {\n\nCONSTEXPR_EXCEPT_WIN_CUDA char div_name[] = \"div_kernel\";\nvoid div_true_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (iter.common_dtype() == kComplexHalf) {\n using scalar_t = c10::complex<at::Half>;\n#if AT_USE_JITERATOR()\n static const auto div_string = jiterator_stringify(\n template <typename T> T div_kernel(T a, T b) { return a / b; });\n opmath_jitted_gpu_kernel_with_scalars<div_name, scalar_t, scalar_t>(\n iter, div_string);\n#else\n using opmath_t = at::opmath_type<scalar_t>;\n opmath_gpu_kernel_with_scalars<scalar_t>(iter, DivFunctor<opmath_t>());\n#endif\n return;\n }\n if (iter.is_cpu_scalar(2)) {\n // optimization for floating-point types: if the second operand is a CPU\n // scalar, compute a * reciprocal(b). Note that this may lose one bit of\n // precision compared to computing the division.\n AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(\n kHalf, kBFloat16, common_dtype, \"div_true_cuda\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto inv_b = opmath_t(1.0) / iter.scalar_value<opmath_t>(2);\n iter.remove_operand(2);\n gpu_kernel(\n iter,\n BUnaryFunctor<scalar_t, scalar_t, scalar_t, MulFunctor<opmath_t>>(\n MulFunctor<opmath_t>(), inv_b));\n });\n } else {\n AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(\n kHalf, kBFloat16, common_dtype, \"div_true_cuda\", [&]() {\n DivFunctor<scalar_t> f;\n gpu_kernel_with_scalars(iter, f);\n });\n }\n}\n} // namespace binary_internal\n\nREGISTER_DISPATCH(div_true_stub, &binary_internal::div_true_kernel_cuda);\n\n} // namespace at::native\n\n\n###",992 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <c10/hip/HIPGuard.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/hip\\BinaryInternal.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\n#include <type_traits>\n\nnamespace at::native {\nnamespace binary_internal {\n\nCONSTEXPR_EXCEPT_WIN_HIP char div_name[] = \"div_kernel\";\nvoid div_true_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (iter.common_dtype() == kComplexHalf) {\n using scalar_t = c10::complex<at::Half>;\n#if AT_USE_JITERATOR()\n static const auto div_string = jiterator_stringify(\n template <typename T> T div_kernel(T a, T b) { return a / b; });\n opmath_jitted_gpu_kernel_with_scalars<div_name, scalar_t, scalar_t>(\n iter, div_string);\n#else\n using opmath_t = at::opmath_type<scalar_t>;\n opmath_gpu_kernel_with_scalars<scalar_t>(iter, DivFunctor<opmath_t>());\n#endif\n return;\n }\n if (iter.is_cpu_scalar(2)) {\n // optimization for floating-point types: if the second operand is a CPU\n // scalar, compute a * reciprocal(b). Note that this may lose one bit of\n // precision compared to computing the division.\n AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(\n kHalf, kBFloat16, common_dtype, \"div_true_hip\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n auto inv_b = opmath_t(1.0) / iter.scalar_value<opmath_t>(2);\n iter.remove_operand(2);\n gpu_kernel(\n iter,\n BUnaryFunctor<scalar_t, scalar_t, scalar_t, MulFunctor<opmath_t>>(\n MulFunctor<opmath_t>(), inv_b));\n });\n } else {\n AT_DISPATCH_FLOATING_AND_COMPLEX_TYPES_AND2(\n kHalf, kBFloat16, common_dtype, \"div_true_hip\", [&]() {\n DivFunctor<scalar_t> f;\n gpu_kernel_with_scalars(iter, f);\n });\n }\n}\n} // namespace binary_internal\n\nREGISTER_DISPATCH(div_true_stub, &binary_internal::div_true_kernel_hip);\n\n} // namespace at::native\n###"993 },994 {995 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <c10/cuda/CUDAGuard.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\n#include <type_traits>\n\nnamespace at::native {\nnamespace binary_internal {\n\nvoid div_trunc_kernel_cuda(TensorIteratorBase& iter) {\n auto dtype = iter.common_dtype();\n if (isIntegralType(dtype, /*includeBool*/ false)) {\n AT_DISPATCH_INTEGRAL_TYPES(dtype, \"div_trunc_cuda\", [&]() {\n gpu_kernel_with_scalars(\n iter,\n [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t { return a / b; });\n });\n } else if (iter.is_cpu_scalar(2)) {\n // optimization for floating-point types: if the second operand is a CPU\n // scalar, compute a * reciprocal(b). Note that this may lose one bit of\n // precision compared to computing the division.\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_trunc_cuda\", [&]() {\n using accscalar_t = at::acc_type<scalar_t, true>;\n auto inv_b = accscalar_t(1.0) / iter.scalar_value<accscalar_t>(2);\n iter.remove_operand(2);\n gpu_kernel(iter, [inv_b] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return std::trunc(a * inv_b);\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_trunc_cuda\", [&]() {\n gpu_kernel_with_scalars(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return std::trunc(a / b);\n });\n });\n }\n}\n} // namespace binary_internal\n\nREGISTER_DISPATCH(div_trunc_stub, &binary_internal::div_trunc_kernel_cuda);\n\n} // namespace at::native\n\n\n###",996 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <c10/hip/HIPGuard.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\n#include <type_traits>\n\nnamespace at::native {\nnamespace binary_internal {\n\nvoid div_trunc_kernel_hip(TensorIteratorBase& iter) {\n auto dtype = iter.common_dtype();\n if (isIntegralType(dtype, /*includeBool*/ false)) {\n AT_DISPATCH_INTEGRAL_TYPES(dtype, \"div_trunc_hip\", [&]() {\n gpu_kernel_with_scalars(\n iter,\n [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t { return a / b; });\n });\n } else if (iter.is_cpu_scalar(2)) {\n // optimization for floating-point types: if the second operand is a CPU\n // scalar, compute a * reciprocal(b). Note that this may lose one bit of\n // precision compared to computing the division.\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_trunc_hip\", [&]() {\n using accscalar_t = at::acc_type<scalar_t, true>;\n auto inv_b = accscalar_t(1.0) / iter.scalar_value<accscalar_t>(2);\n iter.remove_operand(2);\n gpu_kernel(iter, [inv_b] GPU_LAMBDA(scalar_t a) -> scalar_t {\n return std::trunc(a * inv_b);\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n kHalf, kBFloat16, dtype, \"div_trunc_hip\", [&]() {\n gpu_kernel_with_scalars(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return std::trunc(a / b);\n });\n });\n }\n}\n} // namespace binary_internal\n\nREGISTER_DISPATCH(div_trunc_stub, &binary_internal::div_trunc_kernel_hip);\n\n} // namespace at::native\n###"997 },998 {999 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid atan2_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half, at::ScalarType::BFloat16,\n iter.common_dtype(), \"atan2_cuda\",\n [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return ::atan2(a, b);\n });\n });\n}\n\nvoid hypot_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half, at::ScalarType::BFloat16,\n iter.common_dtype(), \"hypot_cuda\",\n [&]() {\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(\n iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return ::hypot(a, b);\n });\n });\n}\n\nREGISTER_DISPATCH(atan2_stub, &atan2_kernel_cuda);\nREGISTER_DISPATCH(hypot_stub, &hypot_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1000 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid atan2_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half, at::ScalarType::BFloat16,\n iter.common_dtype(), \"atan2_hip\",\n [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return ::atan2(a, b);\n });\n });\n}\n\nvoid hypot_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n at::ScalarType::Half, at::ScalarType::BFloat16,\n iter.common_dtype(), \"hypot_hip\",\n [&]() {\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(\n iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return ::hypot(a, b);\n });\n });\n}\n\nREGISTER_DISPATCH(atan2_stub, &atan2_kernel_hip);\nREGISTER_DISPATCH(hypot_stub, &hypot_kernel_hip);\n\n} // namespace at::native\n###"1001 },1002 {1003 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/BinaryInternal.h>\n#include <c10/cuda/CUDAGuard.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n\n#include <type_traits>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nCONSTEXPR_EXCEPT_WIN_CUDA char mul_name[] = \"mul_kernel\";\nvoid mul_kernel_cuda(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (common_dtype == kComplexHalf) {\n using scalar_t = c10::complex<at::Half>;\n#if AT_USE_JITERATOR()\n static const auto mul_string = jiterator_stringify(\n template <typename T> T mul_kernel(T a, T b) { return a * b; });\n opmath_jitted_gpu_kernel_with_scalars<mul_name, scalar_t, scalar_t>(\n iter, mul_string);\n#else\n using opmath_t = at::opmath_type<scalar_t>;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(\n iter, binary_internal::MulFunctor<opmath_t>());\n#endif\n } else {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND3(\n kHalf, kBFloat16, kBool, iter.common_dtype(), \"mul_cuda\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(\n iter, binary_internal::MulFunctor<opmath_t>());\n });\n }\n}\n\nREGISTER_DISPATCH(mul_stub, &mul_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1004 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\BinaryInternal.h>\n#include <c10/hip/HIPGuard.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/TypeSafeSignMath.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n\n#include <type_traits>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nCONSTEXPR_EXCEPT_WIN_HIP char mul_name[] = \"mul_kernel\";\nvoid mul_kernel_hip(TensorIteratorBase& iter) {\n auto common_dtype = iter.common_dtype();\n if (common_dtype == kComplexHalf) {\n using scalar_t = c10::complex<at::Half>;\n#if AT_USE_JITERATOR()\n static const auto mul_string = jiterator_stringify(\n template <typename T> T mul_kernel(T a, T b) { return a * b; });\n opmath_jitted_gpu_kernel_with_scalars<mul_name, scalar_t, scalar_t>(\n iter, mul_string);\n#else\n using opmath_t = at::opmath_type<scalar_t>;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(\n iter, binary_internal::MulFunctor<opmath_t>());\n#endif\n } else {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND3(\n kHalf, kBFloat16, kBool, iter.common_dtype(), \"mul_hip\", [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t>(\n iter, binary_internal::MulFunctor<opmath_t>());\n });\n }\n}\n\nREGISTER_DISPATCH(mul_stub, &mul_kernel_hip);\n\n} // namespace at::native\n###"1005 },1006 {1007 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/cub.cuh>\n#include <ATen/cuda/CUDAConfig.h>\n\nnamespace at {\nnamespace cuda {\nnamespace cub {\n\nnamespace {\ntemplate <typename scalar_t>\nstruct SumOp {\n __device__ scalar_t operator () (scalar_t a, scalar_t b) const {\n return a + b;\n }\n};\n}\n\ntemplate <typename input_t, typename output_t>\nvoid inclusive_sum_truncating(const input_t *input, output_t *output, int64_t num_items) {\n using NO_ROCM(at_cuda_detail)::cub::Sum;\n inclusive_scan(input, output, Sum{}, num_items);\n}\n\ntemplate void inclusive_sum_truncating(const int32_t *input, int32_t *output, int64_t num_items);\ntemplate void inclusive_sum_truncating(const int64_t *input, int64_t *output, int64_t num_items);\ntemplate void inclusive_sum_truncating(const int32_t *input, int64_t *output, int64_t num_items);\n\ntemplate <typename input_t, typename output_t>\nvoid exclusive_sum_in_common_type(const input_t *input, output_t *output, int64_t num_items) {\n using scalar_t = std::common_type_t<input_t, output_t>;\n exclusive_scan(input, output, SumOp<scalar_t>{}, scalar_t(0), num_items);\n}\n\ntemplate void exclusive_sum_in_common_type(const int32_t *input, int32_t *output, int64_t num_items);\ntemplate void exclusive_sum_in_common_type(const int64_t *input, int64_t *output, int64_t num_items);\n\nnamespace {\nstruct CountMaskOp {\n __device__ int64_t operator() (const uint8_t &x) const {\n return x != 0;\n }\n};\n}\n\nvoid mask_exclusive_sum(const uint8_t *mask, int64_t *output_idx, int64_t n) {\n CountMaskOp op{};\n auto iter = NO_ROCM(at_cuda_detail)::cub::TransformInputIterator<\n bool, decltype(op), decltype(mask)>(mask, op);\n exclusive_scan(iter, output_idx, SumOp<int64_t>{}, int64_t{0}, n);\n}\n\n}}} // namespace at::cuda::cub\n\n\n###",1008 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\cub.cuh>\n#include <ATen/hip\\HIPConfig.h>\n\nnamespace at {\nnamespace hip {\nnamespace cub {\n\nnamespace {\ntemplate <typename scalar_t>\nstruct SumOp {\n __device__ scalar_t operator () (scalar_t a, scalar_t b) const {\n return a + b;\n }\n};\n}\n\ntemplate <typename input_t, typename output_t>\nvoid inclusive_sum_truncating(const input_t *input, output_t *output, int64_t num_items) {\n using NO_ROCM(at_hip_detail)::hipcub::Sum;\n inclusive_scan(input, output, Sum{}, num_items);\n}\n\ntemplate void inclusive_sum_truncating(const int32_t *input, int32_t *output, int64_t num_items);\ntemplate void inclusive_sum_truncating(const int64_t *input, int64_t *output, int64_t num_items);\ntemplate void inclusive_sum_truncating(const int32_t *input, int64_t *output, int64_t num_items);\n\ntemplate <typename input_t, typename output_t>\nvoid exclusive_sum_in_common_type(const input_t *input, output_t *output, int64_t num_items) {\n using scalar_t = std::common_type_t<input_t, output_t>;\n exclusive_scan(input, output, SumOp<scalar_t>{}, scalar_t(0), num_items);\n}\n\ntemplate void exclusive_sum_in_common_type(const int32_t *input, int32_t *output, int64_t num_items);\ntemplate void exclusive_sum_in_common_type(const int64_t *input, int64_t *output, int64_t num_items);\n\nnamespace {\nstruct CountMaskOp {\n __device__ int64_t operator() (const uint8_t &x) const {\n return x != 0;\n }\n};\n}\n\nvoid mask_exclusive_sum(const uint8_t *mask, int64_t *output_idx, int64_t n) {\n CountMaskOp op{};\n auto iter = NO_ROCM(at_hip_detail)::hipcub::TransformInputIterator<\n bool, decltype(op), decltype(mask)>(mask, op);\n exclusive_scan(iter, output_idx, SumOp<int64_t>{}, int64_t{0}, n);\n}\n\n}}} // namespace at::cuda::cub\n###"1009 },1010 {1011 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/TensorIterator.h>\n#include <c10/util/TypeSafeSignMath.h>\n\n#include <type_traits>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid remainder_kernel_cuda(TensorIteratorBase& iter) {\n if (isIntegralType(iter.common_dtype(), /*includeBool*/ false)) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"remainder_cuda\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n scalar_t r = a % b;\n if (r != 0 && c10::signs_differ(r, b)) {\n r += b;\n }\n return r;\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, iter.common_dtype(), \"remainder_cuda\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) __ubsan_ignore_float_divide_by_zero__ -> scalar_t {\n auto mod = ::fmod(a, b);\n if (mod != 0 && c10::signs_differ(b, mod)) {\n mod += b;\n }\n return mod;\n });\n });\n }\n}\n\nvoid fmod_kernel_cuda(TensorIteratorBase& iter) {\n if (isIntegralType(iter.common_dtype(), /*includeBool*/ false)) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"fmod_cuda\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return a % b;\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, iter.common_dtype(), \"fmod_cuda\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) __ubsan_ignore_float_divide_by_zero__ -> scalar_t {\n return ::fmod(a, b);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(remainder_stub, &remainder_kernel_cuda);\nREGISTER_DISPATCH(fmod_stub, &fmod_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1012 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/TensorIterator.h>\n#include <c10/util/TypeSafeSignMath.h>\n\n#include <type_traits>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid remainder_kernel_hip(TensorIteratorBase& iter) {\n if (isIntegralType(iter.common_dtype(), /*includeBool*/ false)) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"remainder_hip\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n scalar_t r = a % b;\n if (r != 0 && c10::signs_differ(r, b)) {\n r += b;\n }\n return r;\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, iter.common_dtype(), \"remainder_hip\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) __ubsan_ignore_float_divide_by_zero__ -> scalar_t {\n auto mod = ::fmod(a, b);\n if (mod != 0 && c10::signs_differ(b, mod)) {\n mod += b;\n }\n return mod;\n });\n });\n }\n}\n\nvoid fmod_kernel_hip(TensorIteratorBase& iter) {\n if (isIntegralType(iter.common_dtype(), /*includeBool*/ false)) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"fmod_hip\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return a % b;\n });\n });\n } else {\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, iter.common_dtype(), \"fmod_hip\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) __ubsan_ignore_float_divide_by_zero__ -> scalar_t {\n return ::fmod(a, b);\n });\n });\n }\n}\n\nREGISTER_DISPATCH(remainder_stub, &remainder_kernel_hip);\nREGISTER_DISPATCH(fmod_stub, &fmod_kernel_hip);\n\n} // namespace at::native\n###"1013 },1014 {1015 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\n\nvoid lshift_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.dtype(), \"lshift_cuda\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return static_cast<std::make_unsigned_t<scalar_t>>(a) << b;\n });\n });\n}\n\nvoid rshift_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.dtype(), \"rshift_cuda\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return a >> b;\n });\n });\n}\n\nREGISTER_DISPATCH(lshift_stub, &lshift_kernel_cuda);\nREGISTER_DISPATCH(rshift_stub, &rshift_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1016 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\n\nvoid lshift_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.dtype(), \"lshift_hip\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return static_cast<std::make_unsigned_t<scalar_t>>(a) << b;\n });\n });\n}\n\nvoid rshift_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_INTEGRAL_TYPES(iter.dtype(), \"rshift_hip\", [&]() {\n gpu_kernel_with_scalars(iter,\n []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return a >> b;\n });\n });\n}\n\nREGISTER_DISPATCH(lshift_stub, &lshift_kernel_hip);\nREGISTER_DISPATCH(rshift_stub, &rshift_kernel_hip);\n\n} // namespace at::native\n###"1017 },1018 {1019 "cuda": "\n#pragma once\n#include <thrust/tuple.h>\n#include <ATen/native/SharedReduceOps.h>\n#include <ATen/cuda/DeviceUtils.cuh>\nnamespace at {\nnamespace native {\nnamespace cuda_utils {\nconstexpr int kCUDABlockReduceNumThreads = 512;\n\n\n\n\nconstexpr int kCUDABlockReduceMaxThreads = C10_WARP_SIZE * C10_WARP_SIZE;\n\n\n\n\ntemplate <typename T>\n__inline__ __device__ T WarpReduceSum(T val) {\n#pragma unroll\n for (int offset = (C10_WARP_SIZE >> 1); offset > 0; offset >>= 1) {\n val += WARP_SHFL_DOWN(val, offset);\n }\n return val;\n}\nstruct Block1D {\n static __forceinline__ __device__ int Tid() { return threadIdx.x; }\n static __forceinline__ __device__ int Warps() {\n return blockDim.x / C10_WARP_SIZE;\n }\n};\nstruct Block2D {\n static __forceinline__ __device__ int Tid() {\n return threadIdx.x + threadIdx.y * blockDim.x;\n }\n static __forceinline__ __device__ int Warps() {\n return blockDim.x * blockDim.y / C10_WARP_SIZE;\n }\n};\n\n\n\n\n\n\n\ntemplate <typename T, typename B = Block1D>\n__inline__ __device__ T BlockReduceSum(T val, T* shared) {\n const int tid = B::Tid();\n const int lid = tid % C10_WARP_SIZE;\n const int wid = tid / C10_WARP_SIZE;\n val = WarpReduceSum(val);\n __syncthreads(); \n if (lid == 0) {\n shared[wid] = val;\n }\n __syncthreads();\n val = (tid < B::Warps()) ? shared[lid] : T(0);\n if (wid == 0) {\n val = WarpReduceSum(val);\n }\n return val;\n}\ntemplate <typename T, class ReduceOp>\n__inline__ __device__ T WarpReduce(T val, const ReduceOp& op) {\n#pragma unroll\n for (int offset = (C10_WARP_SIZE >> 1); offset > 0; offset >>= 1) {\n val = op.combine(val, op.warp_shfl_down(val, offset));\n }\n return val;\n}\ntemplate <typename T, class ReduceOp, typename B = Block1D>\n__inline__ __device__ T\nBlockReduce(T val, const ReduceOp& op, const T& identity_element, T* shared) {\n const int tid = B::Tid();\n const int lid = tid % C10_WARP_SIZE;\n const int wid = tid / C10_WARP_SIZE;\n val = WarpReduce(val, op);\n __syncthreads(); \n if (lid == 0) {\n shared[wid] = val;\n }\n __syncthreads();\n val = (tid < B::Warps()) ? shared[lid] : identity_element;\n if (wid == 0) {\n val = WarpReduce(val, op);\n }\n return val;\n}\n} \n} \n} \n\n###",1020 "hip": " \n#include \"hip/hip_runtime.h\"\n#pragma once\n#include <thrust/tuple.h>\n#include <ATen/native/SharedReduceOps.h>\n#include <ATen/hip\\DeviceUtils.cuh>\nnamespace at {\nnamespace native {\nnamespace hip_utils {\nconstexpr int kHIPBlockReduceNumThreads = 512;\n\n\n\n\nconstexpr int kHIPBlockReduceMaxThreads = C10_WARP_SIZE * C10_WARP_SIZE;\n\n\n\n\ntemplate <typename T>\n__inline__ __device__ T WarpReduceSum(T val) {\n#pragma unroll\n for (int offset = (C10_WARP_SIZE >> 1); offset > 0; offset >>= 1) {\n val += WARP_SHFL_DOWN(val, offset);\n }\n return val;\n}\nstruct Block1D {\n static __forceinline__ __device__ int Tid() { return threadIdx.x; }\n static __forceinline__ __device__ int Warps() {\n return blockDim.x / C10_WARP_SIZE;\n }\n};\nstruct Block2D {\n static __forceinline__ __device__ int Tid() {\n return threadIdx.x + threadIdx.y * blockDim.x;\n }\n static __forceinline__ __device__ int Warps() {\n return blockDim.x * blockDim.y / C10_WARP_SIZE;\n }\n};\n\n\n\n\n\n\n\ntemplate <typename T, typename B = Block1D>\n__inline__ __device__ T BlockReduceSum(T val, T* shared) {\n const int tid = B::Tid();\n const int lid = tid % C10_WARP_SIZE;\n const int wid = tid / C10_WARP_SIZE;\n val = WarpReduceSum(val);\n __syncthreads(); \n if (lid == 0) {\n shared[wid] = val;\n }\n __syncthreads();\n val = (tid < B::Warps()) ? shared[lid] : T(0);\n if (wid == 0) {\n val = WarpReduceSum(val);\n }\n return val;\n}\ntemplate <typename T, class ReduceOp>\n__inline__ __device__ T WarpReduce(T val, const ReduceOp& op) {\n#pragma unroll\n for (int offset = (C10_WARP_SIZE >> 1); offset > 0; offset >>= 1) {\n val = op.combine(val, op.warp_shfl_down(val, offset));\n }\n return val;\n}\ntemplate <typename T, class ReduceOp, typename B = Block1D>\n__inline__ __device__ T\nBlockReduce(T val, const ReduceOp& op, const T& identity_element, T* shared) {\n const int tid = B::Tid();\n const int lid = tid % C10_WARP_SIZE;\n const int wid = tid / C10_WARP_SIZE;\n val = WarpReduce(val, op);\n __syncthreads(); \n if (lid == 0) {\n shared[wid] = val;\n }\n __syncthreads();\n val = (tid < B::Warps()) ? shared[lid] : identity_element;\n if (wid == 0) {\n val = WarpReduce(val, op);\n }\n return val;\n}\n} \n} \n} ###"1021 },1022 {1023 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char chebyshev_polynomial_t_name[] = \"chebyshev_polynomial_t_forward\";\n\n void chebyshev_polynomial_t_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_t_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_t_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_t_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_t_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_t_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_t_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_t_stub, &chebyshev_polynomial_t_kernel_cuda);\n} // namespace at::native\n\n\n###",1024 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char chebyshev_polynomial_t_name[] = \"chebyshev_polynomial_t_forward\";\n\n void chebyshev_polynomial_t_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_t_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_t_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_t_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_t_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_t_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_t_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_t_stub, &chebyshev_polynomial_t_kernel_hip);\n} // namespace at::native\n###"1025 },1026 {1027 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char chebyshev_polynomial_u_name[] = \"chebyshev_polynomial_u_forward\";\n\n void chebyshev_polynomial_u_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_u_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_u_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_u_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_u_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_u_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_u_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_u_stub, &chebyshev_polynomial_u_kernel_cuda);\n} // namespace at::native\n\n\n###",1028 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char chebyshev_polynomial_u_name[] = \"chebyshev_polynomial_u_forward\";\n\n void chebyshev_polynomial_u_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_u_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_u_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_u_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_u_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_u_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_u_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_u_stub, &chebyshev_polynomial_u_kernel_hip);\n} // namespace at::native\n###"1029 },1030 {1031 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char chebyshev_polynomial_v_name[] = \"chebyshev_polynomial_v_forward\";\n\n void chebyshev_polynomial_v_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_v_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_v_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_v_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_v_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_v_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_v_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_v_stub, &chebyshev_polynomial_v_kernel_cuda);\n} // namespace at::native\n\n\n###",1032 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char chebyshev_polynomial_v_name[] = \"chebyshev_polynomial_v_forward\";\n\n void chebyshev_polynomial_v_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_v_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_v_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_v_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_v_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_v_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_v_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_v_stub, &chebyshev_polynomial_v_kernel_hip);\n} // namespace at::native\n###"1033 },1034 {1035 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char chebyshev_polynomial_w_name[] = \"chebyshev_polynomial_w_forward\";\n\n void chebyshev_polynomial_w_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_w_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_w_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_w_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_w_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_w_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_w_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_w_stub, &chebyshev_polynomial_w_kernel_cuda);\n} // namespace at::native\n\n\n###",1036 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char chebyshev_polynomial_w_name[] = \"chebyshev_polynomial_w_forward\";\n\n void chebyshev_polynomial_w_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_w_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<chebyshev_polynomial_w_name, scalar_t, scalar_t>(iterator, chebyshev_polynomial_w_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"chebyshev_polynomial_w_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return chebyshev_polynomial_w_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // chebyshev_polynomial_w_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(chebyshev_polynomial_w_stub, &chebyshev_polynomial_w_kernel_hip);\n} // namespace at::native\n###"1037 },1038 {1039 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/Loops.cuh>\n\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native { namespace {\n\nenum class EqOpType {EQ, NE};\n\ntemplate<typename scalar_t>\nstruct CompareEqFunctor{\n CompareEqFunctor(EqOpType op): op_(op) {}\n const EqOpType op_;\n __device__ __forceinline__ bool operator() (scalar_t a, scalar_t b) const {\n if (op_ == EqOpType::EQ) {\n return a == b;\n } else { //NE\n return a != b;\n }\n\n }\n };\n}\n\nC10_NOINLINE void compare_eq_ne_kernel(TensorIteratorBase &iter, EqOpType op) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4(kComplexHalf, kHalf, kBFloat16, kBool,\n iter.common_dtype(), \"compare_eq_ne_cuda\", [&]() {\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t, bool>(\n iter, CompareEqFunctor<scalar_t>(op));\n });\n}\n\nvoid eq_kernel_cuda(TensorIteratorBase& iter) {\n compare_eq_ne_kernel(iter, EqOpType::EQ);\n}\n\nvoid ne_kernel_cuda(TensorIteratorBase& iter) {\n compare_eq_ne_kernel(iter, EqOpType::NE);\n}\n\nREGISTER_DISPATCH(eq_stub, &eq_kernel_cuda);\nREGISTER_DISPATCH(ne_stub, &ne_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1040 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\Loops.cuh>\n\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native { namespace {\n\nenum class EqOpType {EQ, NE};\n\ntemplate<typename scalar_t>\nstruct CompareEqFunctor{\n CompareEqFunctor(EqOpType op): op_(op) {}\n const EqOpType op_;\n __device__ __forceinline__ bool operator() (scalar_t a, scalar_t b) const {\n if (op_ == EqOpType::EQ) {\n return a == b;\n } else { //NE\n return a != b;\n }\n\n }\n };\n}\n\nC10_NOINLINE void compare_eq_ne_kernel(TensorIteratorBase &iter, EqOpType op) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4(kComplexHalf, kHalf, kBFloat16, kBool,\n iter.common_dtype(), \"compare_eq_ne_hip\", [&]() {\n opmath_symmetric_gpu_kernel_with_scalars<scalar_t, bool>(\n iter, CompareEqFunctor<scalar_t>(op));\n });\n}\n\nvoid eq_kernel_hip(TensorIteratorBase& iter) {\n compare_eq_ne_kernel(iter, EqOpType::EQ);\n}\n\nvoid ne_kernel_hip(TensorIteratorBase& iter) {\n compare_eq_ne_kernel(iter, EqOpType::NE);\n}\n\nREGISTER_DISPATCH(eq_stub, &eq_kernel_hip);\nREGISTER_DISPATCH(ne_stub, &ne_kernel_hip);\n\n} // namespace at::native\n###"1041 },1042 {1043 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/TensorFactories.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/Loops.cuh>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\nnamespace {\n\nvoid complex_kernel_cuda(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND(kHalf, iter.input_dtype(0), \"complex_cuda\", [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> c10::complex<scalar_t> {\n return c10::complex<scalar_t>(a, b);\n });\n });\n}\n\nvoid polar_kernel_cuda(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES(iter.input_dtype(0), \"polar_cuda\", [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> c10::complex<scalar_t> {\n return c10::complex<scalar_t>(a * std::cos(b), a * std::sin(b));\n });\n });\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(complex_stub, &complex_kernel_cuda);\nREGISTER_DISPATCH(polar_stub, &polar_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1044 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/TensorFactories.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\Loops.cuh>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\nnamespace {\n\nvoid complex_kernel_hip(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND(kHalf, iter.input_dtype(0), \"complex_hip\", [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> c10::complex<scalar_t> {\n return c10::complex<scalar_t>(a, b);\n });\n });\n}\n\nvoid polar_kernel_hip(TensorIterator& iter) {\n AT_DISPATCH_FLOATING_TYPES(iter.input_dtype(0), \"polar_hip\", [&]() {\n gpu_kernel(\n iter, [] GPU_LAMBDA(scalar_t a, scalar_t b) -> c10::complex<scalar_t> {\n return c10::complex<scalar_t>(a * std::cos(b), a * std::sin(b));\n });\n });\n}\n\n} // anonymous namespace\n\nREGISTER_DISPATCH(complex_stub, &complex_kernel_hip);\nREGISTER_DISPATCH(polar_stub, &polar_kernel_hip);\n\n} // namespace at::native\n###"1045 },1046 {1047 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n#if defined(__CUDACC__)\n#include <cuda.h>\n#include <cuda_fp16.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#elif defined(__HIPCC__)\n#include <hip/hip_runtime.h>\n#include <hip/hip_fp16.h>\n#include <c10/hip/HIPMathCompat.h>\n#endif\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid copysign_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(kBFloat16, kHalf, iter.common_dtype(), \"copysign_cuda\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return c10::cuda::compat::copysign(a, b);\n });\n });\n}\n\nREGISTER_DISPATCH(copysign_stub, ©sign_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1048 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n\n#if defined(__HIPCC__)\n#include <hip/hip_runtime.h>\n#include <hip/hip_fp16.h>\n#include <c10/hip/HIPMathCompat.h>\n#elif defined(__HIPCC__)\n#include <hip/hip_runtime.h>\n#include <hip/hip_fp16.h>\n#include <c10/hip/HIPMathCompat.h>\n#endif\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid copysign_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(kBFloat16, kHalf, iter.common_dtype(), \"copysign_hip\", [&]() {\n gpu_kernel_with_scalars(iter, []GPU_LAMBDA(scalar_t a, scalar_t b) -> scalar_t {\n return c10::hip::compat::copysign(a, b);\n });\n });\n}\n\nREGISTER_DISPATCH(copysign_stub, ©sign_kernel_hip);\n\n} // namespace at::native\n###"1049 },1050 {1051 "cuda": "\n#pragma once\n\n#if !defined(USE_ROCM)\n#include <cuda.h> // for CUDA_VERSION\n#endif\n\n#if !defined(USE_ROCM)\n#include <cub/version.cuh>\n#else\n#define CUB_VERSION 0\n#endif\n\n// cub sort support for __nv_bfloat16 is added to cub 1.13 in:\n// https://github.com/NVIDIA/cub/pull/306\n#if CUB_VERSION >= 101300\n#define CUB_SUPPORTS_NV_BFLOAT16() true\n#else\n#define CUB_SUPPORTS_NV_BFLOAT16() false\n#endif\n\n// cub support for CUB_WRAPPED_NAMESPACE is added to cub 1.13.1 in:\n// https://github.com/NVIDIA/cub/pull/326\n// CUB_WRAPPED_NAMESPACE is defined globally in cmake/Dependencies.cmake\n// starting from CUDA 11.5\n#if defined(CUB_WRAPPED_NAMESPACE) || defined(THRUST_CUB_WRAPPED_NAMESPACE)\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() true\n#else\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() false\n#endif\n\n// cub support for UniqueByKey is added to cub 1.16 in:\n// https://github.com/NVIDIA/cub/pull/405\n#if CUB_VERSION >= 101600\n#define CUB_SUPPORTS_UNIQUE_BY_KEY() true\n#else\n#define CUB_SUPPORTS_UNIQUE_BY_KEY() false\n#endif\n\n// cub support for scan by key is added to cub 1.15\n// in https://github.com/NVIDIA/cub/pull/376\n#if CUB_VERSION >= 101500\n#define CUB_SUPPORTS_SCAN_BY_KEY() 1\n#else\n#define CUB_SUPPORTS_SCAN_BY_KEY() 0\n#endif\n\n// cub support for cub::FutureValue is added to cub 1.15 in:\n// https://github.com/NVIDIA/cub/pull/305\n#if CUB_VERSION >= 101500\n#define CUB_SUPPORTS_FUTURE_VALUE() true\n#else\n#define CUB_SUPPORTS_FUTURE_VALUE() false\n#endif\n\n\n###",1052 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#if !defined(USE_ROCM)\n#include <hip/hip_runtime.h> // for TORCH_HIP_VERSION\n#endif\n\n#if !defined(USE_ROCM)\n#include <cub/version.cuh>\n#else\n#define CUB_VERSION 0\n#endif\n\n// cub sort support for __nv_bfloat16 is added to cub 1.13 in:\n// https://github.com/NVIDIA/cub/pull/306\n#if CUB_VERSION >= 101300\n#define CUB_SUPPORTS_NV_BFLOAT16() true\n#else\n#define CUB_SUPPORTS_NV_BFLOAT16() false\n#endif\n\n// cub support for CUB_WRAPPED_NAMESPACE is added to cub 1.13.1 in:\n// https://github.com/NVIDIA/cub/pull/326\n// CUB_WRAPPED_NAMESPACE is defined globally in cmake/Dependencies.cmake\n// starting from HIP 11.5\n#if defined(CUB_WRAPPED_NAMESPACE) || defined(THRUST_CUB_WRAPPED_NAMESPACE)\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() true\n#else\n#define USE_GLOBAL_CUB_WRAPPED_NAMESPACE() false\n#endif\n\n// cub support for UniqueByKey is added to cub 1.16 in:\n// https://github.com/NVIDIA/cub/pull/405\n#if CUB_VERSION >= 101600\n#define CUB_SUPPORTS_UNIQUE_BY_KEY() true\n#else\n#define CUB_SUPPORTS_UNIQUE_BY_KEY() false\n#endif\n\n// cub support for scan by key is added to cub 1.15\n// in https://github.com/NVIDIA/cub/pull/376\n#if CUB_VERSION >= 101500\n#define CUB_SUPPORTS_SCAN_BY_KEY() 1\n#else\n#define CUB_SUPPORTS_SCAN_BY_KEY() 0\n#endif\n\n// cub support for hipcub::FutureValue is added to cub 1.15 in:\n// https://github.com/NVIDIA/cub/pull/305\n#if CUB_VERSION >= 101500\n#define CUB_SUPPORTS_FUTURE_VALUE() true\n#else\n#define CUB_SUPPORTS_FUTURE_VALUE() false\n#endif\n###"1053 },1054 {1055 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/_local_scalar_dense_native.h>\n#endif\n\n#include <ATen/cuda/CUDAContext.h>\n\nnamespace at::native {\n\nScalar _local_scalar_dense_cuda(const Tensor& self) {\n Scalar r;\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4(\n kComplexHalf, kHalf, kBool, kBFloat16, self.scalar_type(), \"_local_scalar_dense_cuda\", [&] {\n scalar_t value;\n cudaStream_t stream = at::cuda::getCurrentCUDAStream();\n at::cuda::memcpy_and_sync(&value, self.const_data_ptr<scalar_t>(), sizeof(scalar_t), cudaMemcpyDeviceToHost, stream);\n r = Scalar(value);\n });\n return r;\n}\n\n} // at::native\n\n\n###",1056 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/core/Tensor.h>\n#include <ATen/Dispatch.h>\n\n#ifndef AT_PER_OPERATOR_HEADERS\n#include <ATen/NativeFunctions.h>\n#else\n#include <ATen/ops/_local_scalar_dense_native.h>\n#endif\n\n#include <ATen/hip\\HIPContext.h>\n\nnamespace at::native {\n\nScalar _local_scalar_dense_hip(const Tensor& self) {\n Scalar r;\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4(\n kComplexHalf, kHalf, kBool, kBFloat16, self.scalar_type(), \"_local_scalar_dense_hip\", [&] {\n scalar_t value;\n hipStream_t stream = at::hip::getCurrentHIPStream();\n at::cuda::memcpy_and_sync(&value, self.const_data_ptr<scalar_t>(), sizeof(scalar_t), hipMemcpyDeviceToHost, stream);\n r = Scalar(value);\n });\n return r;\n}\n\n} // at::native\n###"1057 },1058 {1059 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/core/TensorBase.h>\n#include <ATen/Dispatch.h>\n\n#include <ATen/native/cuda/ScanKernels.h>\n#include <ATen/native/cuda/ScanUtils.cuh>\n\n#include <limits>\n#include <functional>\n\nnamespace at::native {\n\nvoid launch_cummax_cuda_kernel(const TensorBase& self, const TensorBase& values, const TensorBase& indices, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND3(at::ScalarType::Bool, at::ScalarType::Half, at::ScalarType::BFloat16,\n self.scalar_type(), \"cummax_cuda\", [&]() {\n scalar_t init = self.is_floating_point() ? (-1*std::numeric_limits<scalar_t>::infinity()) : std::numeric_limits<scalar_t>::lowest();\n scan_dim_with_indices<scalar_t>(self, values, indices, dim, init, std::greater_equal<scalar_t>());\n });\n}\n\nvoid launch_cummin_cuda_kernel(const TensorBase& self, const TensorBase& values, const TensorBase& indices, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND3(at::ScalarType::Bool, at::ScalarType::Half, at::ScalarType::BFloat16,\n self.scalar_type(), \"cummin_cuda\", [&]() {\n scalar_t init = self.is_floating_point() ? std::numeric_limits<scalar_t>::infinity() : std::numeric_limits<scalar_t>::max();\n scan_dim_with_indices<scalar_t>(self, values, indices, dim, init, std::less_equal<scalar_t>());\n });\n}\n\n} // namespace at::native\n\n\n###",1060 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/core/TensorBase.h>\n#include <ATen/Dispatch.h>\n\n#include <ATen/native/hip\\ScanKernels.h>\n#include <ATen/native/hip\\ScanUtils.cuh>\n\n#include <limits>\n#include <functional>\n\nnamespace at::native {\n\nvoid launch_cummax_hip_kernel(const TensorBase& self, const TensorBase& values, const TensorBase& indices, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND3(at::ScalarType::Bool, at::ScalarType::Half, at::ScalarType::BFloat16,\n self.scalar_type(), \"cummax_hip\", [&]() {\n scalar_t init = self.is_floating_point() ? (-1*std::numeric_limits<scalar_t>::infinity()) : std::numeric_limits<scalar_t>::lowest();\n scan_dim_with_indices<scalar_t>(self, values, indices, dim, init, std::greater_equal<scalar_t>());\n });\n}\n\nvoid launch_cummin_hip_kernel(const TensorBase& self, const TensorBase& values, const TensorBase& indices, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND3(at::ScalarType::Bool, at::ScalarType::Half, at::ScalarType::BFloat16,\n self.scalar_type(), \"cummin_hip\", [&]() {\n scalar_t init = self.is_floating_point() ? std::numeric_limits<scalar_t>::infinity() : std::numeric_limits<scalar_t>::max();\n scan_dim_with_indices<scalar_t>(self, values, indices, dim, init, std::less_equal<scalar_t>());\n });\n}\n\n} // namespace at::native\n###"1061 },1062 {1063 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/core/TensorBase.h>\n#include <ATen/Dispatch.h>\n\n#include <ATen/native/cuda/ScanKernels.h>\n#include <ATen/native/cuda/ScanUtils.cuh>\n\nnamespace at::native {\n\nvoid launch_cumprod_cuda_kernel(const TensorBase& result, const TensorBase& self, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2(\n ScalarType::Half, ScalarType::BFloat16, self.scalar_type(), \"cumprod_cuda\", [&]() {\n scalar_t init = 1;\n scan_dim<scalar_t>(\n self,\n result,\n dim,\n init,\n std::multiplies<scalar_t>());\n });\n}\n\n} // namespace at::native\n\n\n###",1064 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/core/TensorBase.h>\n#include <ATen/Dispatch.h>\n\n#include <ATen/native/hip\\ScanKernels.h>\n#include <ATen/native/hip\\ScanUtils.cuh>\n\nnamespace at::native {\n\nvoid launch_cumprod_hip_kernel(const TensorBase& result, const TensorBase& self, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2(\n ScalarType::Half, ScalarType::BFloat16, self.scalar_type(), \"cumprod_hip\", [&]() {\n scalar_t init = 1;\n scan_dim<scalar_t>(\n self,\n result,\n dim,\n init,\n std::multiplies<scalar_t>());\n });\n}\n\n} // namespace at::native\n###"1065 },1066 {1067 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/core/TensorBase.h>\n#include <ATen/Dispatch.h>\n\n#include <ATen/native/cuda/ScanKernels.h>\n#include <ATen/native/cuda/ScanUtils.cuh>\n\nnamespace at::native {\n\nvoid launch_cumsum_cuda_kernel(const TensorBase& result, const TensorBase& self, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2(\n ScalarType::Half, ScalarType::BFloat16,\n self.scalar_type(), \"cumsum_cuda\",\n [&]() {\n scalar_t init = 0;\n scan_dim<scalar_t>(\n self,\n result,\n dim,\n init,\n std::plus<scalar_t>());\n });\n}\n\n} // namespace at::native\n\n\n###",1068 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/core/TensorBase.h>\n#include <ATen/Dispatch.h>\n\n#include <ATen/native/hip\\ScanKernels.h>\n#include <ATen/native/hip\\ScanUtils.cuh>\n\nnamespace at::native {\n\nvoid launch_cumsum_hip_kernel(const TensorBase& result, const TensorBase& self, int64_t dim) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND2(\n ScalarType::Half, ScalarType::BFloat16,\n self.scalar_type(), \"cumsum_hip\",\n [&]() {\n scalar_t init = 0;\n scan_dim<scalar_t>(\n self,\n result,\n dim,\n init,\n std::plus<scalar_t>());\n });\n}\n\n} // namespace at::native\n###"1069 },1070 {1071 "cuda": "\n#pragma once\n\nnamespace at { namespace native {\n#if defined(USE_ROCM)\n// take these out when ROCm implements std:: math functions\n#include <math.h>\ntemplate <typename scalar_t>\nstatic __forceinline__ __device__ scalar_t device_sqrt(scalar_t val);\n\ntemplate <>\n__forceinline__ __device__ float device_sqrt(float val) {\n return ::sqrtf(val);\n}\n\ntemplate <>\n__forceinline__ __device__ double device_sqrt(double val) {\n return ::sqrt(val);\n}\n#else\ntemplate<typename scalar_t>\n__forceinline__ __device__ double device_sqrt(scalar_t val) {\n return std::sqrt(val);\n}\n#endif\n}}\n\n\n###",1072 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\nnamespace at { namespace native {\n#if defined(USE_ROCM)\n// take these out when ROCm implements std:: math functions\n#include <math.h>\ntemplate <typename scalar_t>\nstatic __forceinline__ __device__ scalar_t device_sqrt(scalar_t val);\n\ntemplate <>\n__forceinline__ __device__ float device_sqrt(float val) {\n return ::sqrtf(val);\n}\n\ntemplate <>\n__forceinline__ __device__ double device_sqrt(double val) {\n return ::sqrt(val);\n}\n#else\ntemplate<typename scalar_t>\n__forceinline__ __device__ double device_sqrt(scalar_t val) {\n return std::sqrt(val);\n}\n#endif\n}}\n###"1073 },1074 {1075 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/cuda/CUDAApplyUtils.cuh>\n#include <ATen/AccumulateType.h>\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\n#include <curand.h>\n#include <curand_kernel.h>\n#include <curand_philox4x32_x.h>\n#include <utility>\n#include <functional>\n\n#include <ATen/native/Distributions.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n\n#include <cstdint>\n#include <limits>\n#include <utility>\n#include <type_traits>\n\nnamespace at::native {\n\nvoid bernoulli_tensor_kernel(const TensorBase &self, const TensorBase &p_, c10::optional<Generator> gen_) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen_, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::bernoulli_kernel(self, p_, generator);\n}\n\nvoid bernoulli_scalar_kernel(const TensorBase &self, double p, c10::optional<Generator> gen) {\n auto iter = TensorIterator::borrowing_nullary_op(self);\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::bernoulli_kernel(iter, p, generator);\n}\n\nREGISTER_DISPATCH(bernoulli_tensor_stub, &bernoulli_tensor_kernel);\nREGISTER_DISPATCH(bernoulli_scalar_stub, &bernoulli_scalar_kernel);\n\n} // namespace at::native\n\n\n###",1076 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/hip\\HIPApplyUtils.cuh>\n#include <ATen/AccumulateType.h>\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\n#include <hiprand/hiprand.h>\n#include <hiprand/hiprand_kernel.h>\n#include <hiprand/hiprand_kernel.h>\n#include <utility>\n#include <functional>\n\n#include <ATen/native/Distributions.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n\n#include <cstdint>\n#include <limits>\n#include <utility>\n#include <type_traits>\n\nnamespace at::native {\n\nvoid bernoulli_tensor_kernel(const TensorBase &self, const TensorBase &p_, c10::optional<Generator> gen_) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen_, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::bernoulli_kernel(self, p_, generator);\n}\n\nvoid bernoulli_scalar_kernel(const TensorBase &self, double p, c10::optional<Generator> gen) {\n auto iter = TensorIterator::borrowing_nullary_op(self);\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::bernoulli_kernel(iter, p, generator);\n}\n\nREGISTER_DISPATCH(bernoulli_tensor_stub, &bernoulli_tensor_kernel);\nREGISTER_DISPATCH(bernoulli_scalar_stub, &bernoulli_scalar_kernel);\n\n} // namespace at::native\n###"1077 },1078 {1079 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid cauchy_kernel(TensorIteratorBase& iter, double median, double sigma, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::cauchy_kernel(iter, median, sigma, generator);\n}\n\nREGISTER_DISPATCH(cauchy_stub, &cauchy_kernel);\n\n} // namespace at::native\n\n\n###",1080 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid cauchy_kernel(TensorIteratorBase& iter, double median, double sigma, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::cauchy_kernel(iter, median, sigma, generator);\n}\n\nREGISTER_DISPATCH(cauchy_stub, &cauchy_kernel);\n\n} // namespace at::native\n###"1081 },1082 {1083 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid exponential_kernel(TensorIteratorBase& iter, double lambda, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::exponential_kernel(iter, lambda, generator);\n}\n\nREGISTER_DISPATCH(exponential_stub, &exponential_kernel);\n\n} // namespace at::native\n\n\n###",1084 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid exponential_kernel(TensorIteratorBase& iter, double lambda, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::exponential_kernel(iter, lambda, generator);\n}\n\nREGISTER_DISPATCH(exponential_stub, &exponential_kernel);\n\n} // namespace at::native\n###"1085 },1086 {1087 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid geometric_kernel(TensorIteratorBase& iter, double p_, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::geometric_kernel(iter, p_, generator);\n}\n\nREGISTER_DISPATCH(geometric_stub, &geometric_kernel);\n\n} // namespace at::native\n\n\n###",1088 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid geometric_kernel(TensorIteratorBase& iter, double p_, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::geometric_kernel(iter, p_, generator);\n}\n\nREGISTER_DISPATCH(geometric_stub, &geometric_kernel);\n\n} // namespace at::native\n###"1089 },1090 {1091 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid log_normal_kernel(TensorIteratorBase& iter, double mean, double std, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::log_normal_kernel(iter, mean, std, generator);\n}\n\nREGISTER_DISPATCH(log_normal_stub, &log_normal_kernel);\n\n} // namespace at::native\n\n\n###",1092 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid log_normal_kernel(TensorIteratorBase& iter, double mean, double std, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::log_normal_kernel(iter, mean, std, generator);\n}\n\nREGISTER_DISPATCH(log_normal_stub, &log_normal_kernel);\n\n} // namespace at::native\n###"1093 },1094 {1095 "cuda": "\n#pragma once\n\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/cuda/CUDAEvent.h>\n#include <ATen/cuda/detail/UnpackRaw.cuh>\n#include <ATen/cuda/detail/CUDAHooks.h>\n#include <ATen/detail/CUDAHooksInterface.h>\n#include <c10/core/StreamGuard.h>\n#include <c10/cuda/CUDAGraphsC10Utils.h>\n#include <c10/cuda/CUDAGuard.h>\n\n// c10/cuda/CUDAGraphsC10Utils.h has utils used by both c10 and aten.\n// This file adds utils used by aten only.\n\nnamespace at {\nnamespace cuda {\n\nusing CaptureId_t = c10::cuda::CaptureId_t;\nusing CaptureStatus = c10::cuda::CaptureStatus;\n\n// Use this version where you don't want to create a CUDA context if none exists.\ninline CaptureStatus currentStreamCaptureStatus() {\n#if !defined(USE_ROCM) || ROCM_VERSION >= 50300\n // don't create a context if we don't have to\n if (c10::cuda::hasPrimaryContext(c10::cuda::current_device())) {\n return c10::cuda::currentStreamCaptureStatusMayInitCtx();\n } else {\n return CaptureStatus::None;\n }\n#else\n return CaptureStatus::None;\n#endif\n}\n\ninline void assertNotCapturing(std::string attempt) {\n auto status = currentStreamCaptureStatus();\n TORCH_CHECK(status == CaptureStatus::None,\n attempt,\n \" during CUDA graph capture. If you need this call to be captured, \"\n \"please file an issue. \"\n \"Current cudaStreamCaptureStatus: \",\n status);\n}\n\ninline void errorIfCapturingCudnnBenchmark(std::string version_specific) {\n auto status = currentStreamCaptureStatus();\n TORCH_CHECK(status == CaptureStatus::None,\n \"Current cudaStreamCaptureStatus: \",\n status,\n \"\\nCapturing \",\n version_specific,\n \"is prohibited. Possible causes of this error:\\n\"\n \"1. No warmup iterations occurred before capture.\\n\"\n \"2. The convolutions you're trying to capture use dynamic shapes, \"\n \"in which case capturing them is generally prohibited.\");\n}\n\n} // namespace cuda\n} // namespace at\n\n\n###",1096 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/hip\\HIPEvent.h>\n#include <ATen/hip/detail\\UnpackRaw.cuh>\n#include <ATen/hip/detail\\HIPHooks.h>\n#include <ATen/detail/HIPHooksInterface.h>\n#include <c10/core/StreamGuard.h>\n#include <c10/hip/HIPGraphsC10Utils.h>\n#include <c10/hip/HIPGuard.h>\n\n// c10/hip/HIPGraphsC10Utils.h has utils used by both c10 and aten.\n// This file adds utils used by aten only.\n\nnamespace at {\nnamespace hip {\n\nusing CaptureId_t = c10::hip::CaptureId_t;\nusing CaptureStatus = c10::hip::CaptureStatus;\n\n// Use this version where you don't want to create a HIP context if none exists.\ninline CaptureStatus currentStreamCaptureStatus() {\n#if !defined(USE_ROCM) || ROCM_VERSION >= 50300\n // don't create a context if we don't have to\n if (c10::hip::hasPrimaryContext(c10::hip::current_device())) {\n return c10::hip::currentStreamCaptureStatusMayInitCtx();\n } else {\n return CaptureStatus::None;\n }\n#else\n return CaptureStatus::None;\n#endif\n}\n\ninline void assertNotCapturing(std::string attempt) {\n auto status = currentStreamCaptureStatus();\n TORCH_CHECK(status == CaptureStatus::None,\n attempt,\n \" during HIP graph capture. If you need this call to be captured, \"\n \"please file an issue. \"\n \"Current hipStreamCaptureStatus: \",\n status);\n}\n\ninline void errorIfCapturingCudnnBenchmark(std::string version_specific) {\n auto status = currentStreamCaptureStatus();\n TORCH_CHECK(status == CaptureStatus::None,\n \"Current hipStreamCaptureStatus: \",\n status,\n \"\\nCapturing \",\n version_specific,\n \"is prohibited. Possible causes of this error:\\n\"\n \"1. No warmup iterations occurred before capture.\\n\"\n \"2. The convolutions you're trying to capture use dynamic shapes, \"\n \"in which case capturing them is generally prohibited.\");\n}\n\n} // namespace hip\n} // namespace at\n###"1097 },1098 {1099 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/native/UnaryOps.h>\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid normal_kernel(const TensorBase &self, double mean, double std, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::normal_kernel(self, mean, std, generator);\n}\n\nREGISTER_DISPATCH(normal_stub, &normal_kernel);\n\n} // namespace at::native\n\n\n###",1100 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/native/UnaryOps.h>\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid normal_kernel(const TensorBase &self, double mean, double std, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::normal_kernel(self, mean, std, generator);\n}\n\nREGISTER_DISPATCH(normal_stub, &normal_kernel);\n\n} // namespace at::native\n###"1101 },1102 {1103 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid random_from_to_kernel(TensorIteratorBase& iter, uint64_t range, int64_t base, c10::optional<Generator> gen_) {\n auto gen = get_generator_or_default<CUDAGeneratorImpl>(gen_, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::random_from_to_kernel(iter, range, base, gen);\n}\n\nvoid random_full_64_bits_range_kernel(TensorIteratorBase& iter, c10::optional<Generator> gen_) {\n auto gen = get_generator_or_default<CUDAGeneratorImpl>(gen_, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::random_full_64_bits_range_kernel(iter, gen);\n}\n\nvoid random_kernel(TensorIteratorBase& iter, c10::optional<Generator> gen_) {\n auto gen = get_generator_or_default<CUDAGeneratorImpl>(gen_, cuda::detail::getDefaultCUDAGenerator());\n at::native::templates::cuda::random_kernel(iter, gen);\n}\n\nREGISTER_DISPATCH(random_from_to_stub, &random_from_to_kernel);\nREGISTER_DISPATCH(random_stub, &random_kernel);\nREGISTER_DISPATCH(random_full_64_bits_range_stub, &random_full_64_bits_range_kernel);\n\n} // namespace at::native\n\n\n###",1104 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid random_from_to_kernel(TensorIteratorBase& iter, uint64_t range, int64_t base, c10::optional<Generator> gen_) {\n auto gen = get_generator_or_default<HIPGeneratorImpl>(gen_, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::random_from_to_kernel(iter, range, base, gen);\n}\n\nvoid random_full_64_bits_range_kernel(TensorIteratorBase& iter, c10::optional<Generator> gen_) {\n auto gen = get_generator_or_default<HIPGeneratorImpl>(gen_, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::random_full_64_bits_range_kernel(iter, gen);\n}\n\nvoid random_kernel(TensorIteratorBase& iter, c10::optional<Generator> gen_) {\n auto gen = get_generator_or_default<HIPGeneratorImpl>(gen_, cuda::detail::getDefaultHIPGenerator());\n at::native::templates::cuda::random_kernel(iter, gen);\n}\n\nREGISTER_DISPATCH(random_from_to_stub, &random_from_to_kernel);\nREGISTER_DISPATCH(random_stub, &random_kernel);\nREGISTER_DISPATCH(random_full_64_bits_range_stub, &random_full_64_bits_range_kernel);\n\n} // namespace at::native\n###"1105 },1106 {1107 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/cuda/CUDAGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/cuda/DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid uniform_kernel(TensorIteratorBase& iter, double from, double to, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<CUDAGeneratorImpl>(gen, cuda::detail::getDefaultCUDAGenerator());\n templates::cuda::uniform_kernel(iter, from, to, generator);\n}\n\nREGISTER_DISPATCH(uniform_stub, &uniform_kernel);\n\n} // namespace at::native\n\n\n###",1108 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/hip\\HIPGeneratorImpl.h>\n#include <ATen/native/UnaryOps.h>\n#include <ATen/native/hip\\DistributionTemplates.h>\n\nnamespace at::native {\n\nvoid uniform_kernel(TensorIteratorBase& iter, double from, double to, c10::optional<Generator> gen) {\n auto generator = get_generator_or_default<HIPGeneratorImpl>(gen, cuda::detail::getDefaultHIPGenerator());\n templates::cuda::uniform_kernel(iter, from, to, generator);\n}\n\nREGISTER_DISPATCH(uniform_stub, &uniform_kernel);\n\n} // namespace at::native\n###"1109 },1110 {1111 "cuda": "\n#pragma once\n#include <ATen/core/Tensor.h>\n#include <ATen/cuda/Atomic.cuh>\n#include <ATen/cuda/CUDAContext.h>\n#include <ATen/TensorUtils.h>\n\nnamespace at {\nnamespace native {\n\nTensor embedding_backward_cuda_kernel(\n const Tensor &grad,\n const Tensor &orig_indices,\n const Tensor &sorted_indices,\n const Tensor &count,\n int64_t num_weights,\n int padding_idx = -1,\n bool mode_mean = false,\n const Tensor &offset2bag = Tensor(),\n const Tensor &bag_size = Tensor(),\n const Tensor &per_sample_weights = Tensor());\n\n}}\n\n\n###",1112 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n#include <ATen/core/Tensor.h>\n#include <ATen/hip\\Atomic.cuh>\n#include <ATen/hip\\HIPContext.h>\n#include <ATen/TensorUtils.h>\n\nnamespace at {\nnamespace native {\n\nTensor embedding_backward_hip_kernel(\n const Tensor &grad,\n const Tensor &orig_indices,\n const Tensor &sorted_indices,\n const Tensor &count,\n int64_t num_weights,\n int padding_idx = -1,\n bool mode_mean = false,\n const Tensor &offset2bag = Tensor(),\n const Tensor &bag_size = Tensor(),\n const Tensor &per_sample_weights = Tensor());\n\n}}\n###"1113 },1114 {1115 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/Fill.h>\n#include <c10/core/Scalar.h>\n\nnamespace at::native {\n\ntemplate<typename scalar_t>\nstruct FillFunctor {\n FillFunctor(scalar_t v): value(v) {}\n __device__ __forceinline__ scalar_t operator() () const {\n return value;\n }\n private:\n scalar_t value;\n};\n\nvoid fill_kernel_cuda(TensorIterator& iter, const Scalar& value) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4(kComplexHalf, kBool, kHalf, kBFloat16, iter.dtype(), \"fill_cuda\", [&]() {\n gpu_kernel(iter, FillFunctor<scalar_t>(value.to<scalar_t>()));\n });\n}\n\nREGISTER_DISPATCH(fill_stub, &fill_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1116 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/Fill.h>\n#include <c10/core/Scalar.h>\n\nnamespace at::native {\n\ntemplate<typename scalar_t>\nstruct FillFunctor {\n FillFunctor(scalar_t v): value(v) {}\n __device__ __forceinline__ scalar_t operator() () const {\n return value;\n }\n private:\n scalar_t value;\n};\n\nvoid fill_kernel_hip(TensorIterator& iter, const Scalar& value) {\n AT_DISPATCH_ALL_TYPES_AND_COMPLEX_AND4(kComplexHalf, kBool, kHalf, kBFloat16, iter.dtype(), \"fill_hip\", [&]() {\n gpu_kernel(iter, FillFunctor<scalar_t>(value.to<scalar_t>()));\n });\n}\n\nREGISTER_DISPATCH(fill_stub, &fill_kernel_hip);\n\n} // namespace at::native\n###"1117 },1118 {1119 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/native/sparse/SparseStubs.h>\n#include <ATen/native/sparse/FlattenIndicesCommon.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/KernelUtils.cuh>\n#include <ATen/cuda/detail/OffsetCalculator.cuh>\n#include <ATen/AccumulateType.h>\n\nnamespace at::native {\n\nnamespace {\n\ntemplate <typename func_t>\nstruct CUDAKernelLauncher {\n static void launch(TensorIteratorBase& iter, const func_t& f) {\n gpu_kernel(iter, f);\n }\n};\n\nTensor flatten_indices_cuda_kernel(const Tensor& indices, IntArrayRef size) {\n return _flatten_indices<CUDAKernelLauncher>(indices, size);\n}\n\n}\n\nREGISTER_CUDA_DISPATCH(flatten_indices_stub, &flatten_indices_cuda_kernel);\n\n} // namespace at::native\n\n\n###",1120 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/native/sparse/SparseStubs.h>\n#include <ATen/native/sparse/FlattenIndicesCommon.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\KernelUtils.cuh>\n#include <ATen/hip/detail\\OffsetCalculator.cuh>\n#include <ATen/AccumulateType.h>\n\nnamespace at::native {\n\nnamespace {\n\ntemplate <typename func_t>\nstruct HIPKernelLauncher {\n static void launch(TensorIteratorBase& iter, const func_t& f) {\n gpu_kernel(iter, f);\n }\n};\n\nTensor flatten_indices_hip_kernel(const Tensor& indices, IntArrayRef size) {\n return _flatten_indices<HIPKernelLauncher>(indices, size);\n}\n\n}\n\nREGISTER_HIP_DISPATCH(flatten_indices_stub, &flatten_indices_hip_kernel);\n\n} // namespace at::native\n###"1121 },1122 {1123 "cuda": "\n#pragma once\n\n#include <ATen/NumericUtils.h>\n\nnamespace at::native {\n\n// std:: does not have clamp functors\ntemplate <typename T>\nstruct minimum {\n __device__ T operator()(const T& a, const T& b) const {\n return (_isnan(a) || a < b) ? a : b;\n }\n};\n\ntemplate <typename T>\nstruct maximum {\n __device__ T operator()(const T& a, const T& b) const {\n return (_isnan(a) || a > b) ? a : b;\n }\n};\n\n} // namespace at::native\n\n\n###",1124 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <ATen/NumericUtils.h>\n\nnamespace at::native {\n\n// std:: does not have clamp functors\ntemplate <typename T>\nstruct minimum {\n __device__ T operator()(const T& a, const T& b) const {\n return (_isnan(a) || a < b) ? a : b;\n }\n};\n\ntemplate <typename T>\nstruct maximum {\n __device__ T operator()(const T& a, const T& b) const {\n return (_isnan(a) || a > b) ? a : b;\n }\n};\n\n} // namespace at::native\n###"1125 },1126 {1127 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/TypeDefault.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/cuda/fused_adam_amsgrad_impl.cuh>\n#include <ATen/native/cuda/fused_adam_impl.cuh>\n#include <c10/util/Exception.h>\n\n\nnamespace at::native {\n\n// note(crcrpar): To observe the CI rules, i.e. 20 minutes per file to compile, defensively split instantiations into _impl files.\n// this is only for CUDA 11.3 for which it took about 20 minutes and 28 minutes in my workstation and CI, respectively.\n// As a data point, it took about 20 seconds for CUDA 11.7 installed in my environment.\n// See https://github.com/pytorch/pytorch/pull/81705 for details.\nvoid _fused_adam_kernel_cuda_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool amsgrad,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n if (amsgrad) {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs}),\n \"params, grads, exp_avgs, exp_avg_sqs, and max_exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adam_amsgrad_cuda_impl_(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n } else {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs}),\n \"params, grads, exp_avgs, and exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adam_cuda_impl_(params, grads, exp_avgs, exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n }\n}\n\n} // namespace at::native\n\n\n###",1128 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/TypeDefault.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/hip\\fused_adam_amsgrad_impl.cuh>\n#include <ATen/native/hip\\fused_adam_impl.cuh>\n#include <c10/util/Exception.h>\n\n\nnamespace at::native {\n\n// note(crcrpar): To observe the CI rules, i.e. 20 minutes per file to compile, defensively split instantiations into _impl files.\n// this is only for HIP 11.3 for which it took about 20 minutes and 28 minutes in my workstation and CI, respectively.\n// As a data point, it took about 20 seconds for HIP 11.7 installed in my environment.\n// See https://github.com/pytorch/pytorch/pull/81705 for details.\nvoid _fused_adam_kernel_hip_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool amsgrad,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n if (amsgrad) {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs}),\n \"params, grads, exp_avgs, exp_avg_sqs, and max_exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adam_amsgrad_hip_impl_(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n } else {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs}),\n \"params, grads, exp_avgs, and exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adam_hip_impl_(params, grads, exp_avgs, exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n }\n}\n\n} // namespace at::native\n###"1129 },1130 {1131 "cuda": "\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/TypeDefault.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/cuda/fused_adamw_amsgrad_impl.cuh>\n#include <ATen/native/cuda/fused_adamw_impl.cuh>\n#include <c10/util/Exception.h>\n\n\nnamespace at { namespace native {\n\n// note(crcrpar): To observe the CI rules, i.e. 20 minutes per file to compile, defensively split instantiations into _impl files.\n// this is only for CUDA 11.3 for which it took about 20 minutes and 28 minutes in my workstation and CI, respectively.\n// As a data point, it took about 20 seconds for CUDA 11.7 installed in my environment.\n// See https://github.com/pytorch/pytorch/pull/81705 for details.\nvoid _fused_adamw_kernel_cuda_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool amsgrad,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n if (amsgrad) {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs}),\n \"params, grads, exp_avgs, exp_avg_sqs, and max_exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adamw_amsgrad_cuda_impl_(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n } else {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs}),\n \"params, grads, exp_avgs, and exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adamw_cuda_impl_(params, grads, exp_avgs, exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n }\n}\n\n}} // namespace at::native\n\n\n###",1132 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_ONLY_METHOD_OPERATORS\n#include <ATen/TypeDefault.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/hip\\fused_adamw_amsgrad_impl.cuh>\n#include <ATen/native/hip\\fused_adamw_impl.cuh>\n#include <c10/util/Exception.h>\n\n\nnamespace at { namespace native {\n\n// note(crcrpar): To observe the CI rules, i.e. 20 minutes per file to compile, defensively split instantiations into _impl files.\n// this is only for HIP 11.3 for which it took about 20 minutes and 28 minutes in my workstation and CI, respectively.\n// As a data point, it took about 20 seconds for HIP 11.7 installed in my environment.\n// See https://github.com/pytorch/pytorch/pull/81705 for details.\nvoid _fused_adamw_kernel_hip_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool amsgrad,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n if (amsgrad) {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs}),\n \"params, grads, exp_avgs, exp_avg_sqs, and max_exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adamw_amsgrad_hip_impl_(params, grads, exp_avgs, exp_avg_sqs, max_exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n } else {\n TORCH_CHECK(\n at::native::check_fast_path_restrictions({params, grads, exp_avgs, exp_avg_sqs}),\n \"params, grads, exp_avgs, and exp_avg_sqs must have same dtype, device, and layout\");\n _fused_adamw_hip_impl_(params, grads, exp_avgs, exp_avg_sqs, state_steps, lr, beta1, beta2, weight_decay, eps, maximize, grad_scale, found_inf);\n }\n}\n\n}} // namespace at::native\n###"1133 },1134 {1135 "cuda": "\n#include <ATen/native/cuda/fused_adamw_amsgrad_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/cuda/fused_adam_utils.cuh>\n#include <ATen/native/cuda/MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_amsgrad_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec(), max_exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adamw_kernel_cuda\", [&]() {\n multi_tensor_apply_for_fused_optimizer<5>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 5>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */true,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ADAMW);\n });\n}\n\n} } // namespace at::native\n\n\n###",1136 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/native/hip\\fused_adamw_amsgrad_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/hip\\fused_adam_utils.cuh>\n#include <ATen/native/hip\\MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_amsgrad_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec(), max_exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adamw_kernel_hip\", [&]() {\n multi_tensor_apply_for_fused_optimizer<5>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 5>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */true,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ADAMW);\n });\n}\n\n} } // namespace at::native\n###"1137 },1138 {1139 "cuda": "\n#pragma once\n\n#include <ATen/Tensor.h>\n#include <c10/util/Half.h>\n\n#include <cuda.h>\n#include <cuda_runtime.h>\n#include <cuda_fp16.h>\n\nnamespace at {\ntemplate <>\ninline __half* Tensor::data() const {\n return reinterpret_cast<__half*>(data<Half>());\n}\n} // namespace at\n\n\n###",1140 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n\n#include <ATen/Tensor.h>\n#include <c10/util/Half.h>\n\n#include <hip/hip_runtime.h>\n#include <hip/hip_runtime.h>\n#include <hip/hip_fp16.h>\n\nnamespace at {\ntemplate <>\ninline __half* Tensor::data() const {\n return reinterpret_cast<__half*>(data<Half>());\n}\n} // namespace at\n###"1141 },1142 {1143 "cuda": "\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_amsgrad_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n\n\n###",1144 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_amsgrad_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n###"1145 },1146 {1147 "cuda": "\n#include <ATen/native/cuda/fused_adamw_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/cuda/fused_adam_utils.cuh>\n#include <ATen/native/cuda/MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adamw_kernel_cuda\", [&]() {\n multi_tensor_apply_for_fused_optimizer<4>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 4>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */false,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ADAMW);\n });\n}\n\n} } // namespace at::native\n\n\n###",1148 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/native/hip\\fused_adamw_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/hip\\fused_adam_utils.cuh>\n#include <ATen/native/hip\\MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adamw_kernel_hip\", [&]() {\n multi_tensor_apply_for_fused_optimizer<4>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 4>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */false,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ADAMW);\n });\n}\n\n} } // namespace at::native\n###"1149 },1150 {1151 "cuda": "\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n\n\n###",1152 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adamw_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n###"1153 },1154 {1155 "cuda": "\n#include <ATen/native/cuda/fused_adam_amsgrad_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/cuda/fused_adam_utils.cuh>\n#include <ATen/native/cuda/MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at::native {\n\nvoid _fused_adam_amsgrad_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec(), max_exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adam_kernel_cuda\", [&]() {\n multi_tensor_apply_for_fused_optimizer<5>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 5>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */true,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ORIGINAL);\n });\n}\n\n} // namespace at::native\n\n\n###",1156 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/native/hip\\fused_adam_amsgrad_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/hip\\fused_adam_utils.cuh>\n#include <ATen/native/hip\\MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at::native {\n\nvoid _fused_adam_amsgrad_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec(), max_exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adam_kernel_hip\", [&]() {\n multi_tensor_apply_for_fused_optimizer<5>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 5>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */true,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ORIGINAL);\n });\n}\n\n} // namespace at::native\n###"1157 },1158 {1159 "cuda": "\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adam_amsgrad_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n\n\n###",1160 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adam_amsgrad_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList max_exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n###"1161 },1162 {1163 "cuda": "\n#include <ATen/native/cuda/fused_adam_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/cuda/fused_adam_utils.cuh>\n#include <ATen/native/cuda/MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at::native {\n\nvoid _fused_adam_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adam_kernel_cuda\", [&]() {\n multi_tensor_apply_for_fused_optimizer<4>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 4>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */false,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ORIGINAL);\n });\n}\n\n} // namespace at::native\n\n\n###",1164 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include <ATen/native/hip\\fused_adam_impl.cuh>\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/ForeachUtils.h>\n#include <ATen/native/hip\\fused_adam_utils.cuh>\n#include <ATen/native/hip\\MultiTensorApply.cuh>\n#include <vector>\n\nnamespace at::native {\n\nvoid _fused_adam_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n) {\n std::vector<std::vector<at::Tensor>> tensor_lists{\n params.vec(), grads.vec(), exp_avgs.vec(), exp_avg_sqs.vec() };\n\n float* grad_scale_ptr = grad_scale.has_value() ? grad_scale->data_ptr<float>() : nullptr;\n float* found_inf_ptr = found_inf.has_value() ? found_inf->data_ptr<float>() : nullptr;\n\n AT_DISPATCH_FLOATING_TYPES_AND2(kHalf, kBFloat16, params[0].scalar_type(),\n \"fused_adam_kernel_hip\", [&]() {\n multi_tensor_apply_for_fused_optimizer<4>(\n tensor_lists,\n state_steps,\n FusedAdamMathFunctor<scalar_t, 4>(),\n lr,\n beta1,\n beta2,\n weight_decay,\n eps,\n maximize,\n /* amsgrad */false,\n grad_scale_ptr,\n found_inf_ptr,\n ADAM_MODE::ORIGINAL);\n });\n}\n\n} // namespace at::native\n###"1165 },1166 {1167 "cuda": "\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adam_cuda_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n\n\n###",1168 "hip": " // !!! This is a file automatically generated by hipify!!!\n#pragma once\n#include <ATen/core/Tensor.h>\n\nnamespace at { namespace native {\n\nvoid _fused_adam_hip_impl_(\n at::TensorList params,\n at::TensorList grads,\n at::TensorList exp_avgs,\n at::TensorList exp_avg_sqs,\n at::TensorList state_steps,\n const double lr,\n const double beta1,\n const double beta2,\n const double weight_decay,\n const double eps,\n const bool maximize,\n const c10::optional<at::Tensor>& grad_scale,\n const c10::optional<at::Tensor>& found_inf\n);\n\n} } // namespace at::native\n###"1169 },1170 {1171 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/cuda/jit_utils.h>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\n// See note [Jiterator]\nCONSTEXPR_EXCEPT_WIN_CUDA char gcd_name[] = \"gcd\";\nvoid gcd_kernel_cuda(TensorIteratorBase& iter) {\n #if AT_USE_JITERATOR()\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"gcd_cuda\", [&]() {\n jitted_gpu_kernel</*name=*/gcd_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 2>(iter, gcd_string);\n });\n #else\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"gcd_cuda\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA (scalar_t a, scalar_t b) -> scalar_t {\n return calc_gcd(a, b);\n });\n });\n #endif // AT_USE_JITERATOR()\n}\n\n// See note [Jiterator]\nCONSTEXPR_EXCEPT_WIN_CUDA char lcm_name[] = \"lcm\";\nvoid lcm_kernel_cuda(TensorIteratorBase& iter) {\n #if AT_USE_JITERATOR()\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"lcm_cuda\", [&]() {\n jitted_gpu_kernel</*name=*/lcm_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 2>(iter, lcm_string);\n });\n #else\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"lcm_cuda\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA (scalar_t a, scalar_t b) -> scalar_t {\n scalar_t g = calc_gcd(a, b);\n return (g == 0) ? 0 : ::abs(a / g * b);\n });\n });\n #endif // AT_USE_JITERATOR()\n}\n\nREGISTER_DISPATCH(gcd_stub, &gcd_kernel_cuda);\nREGISTER_DISPATCH(lcm_stub, &lcm_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1172 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/hip\\jit_utils.h>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\n// See note [Jiterator]\nCONSTEXPR_EXCEPT_WIN_HIP char gcd_name[] = \"gcd\";\nvoid gcd_kernel_hip(TensorIteratorBase& iter) {\n #if AT_USE_JITERATOR()\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"gcd_hip\", [&]() {\n jitted_gpu_kernel</*name=*/gcd_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 2>(iter, gcd_string);\n });\n #else\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"gcd_hip\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA (scalar_t a, scalar_t b) -> scalar_t {\n return calc_gcd(a, b);\n });\n });\n #endif // AT_USE_JITERATOR()\n}\n\n// See note [Jiterator]\nCONSTEXPR_EXCEPT_WIN_HIP char lcm_name[] = \"lcm\";\nvoid lcm_kernel_hip(TensorIteratorBase& iter) {\n #if AT_USE_JITERATOR()\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"lcm_hip\", [&]() {\n jitted_gpu_kernel</*name=*/lcm_name,\n /*return_dtype=*/ scalar_t,\n /*common_dtype=*/ scalar_t,\n /*arity=*/ 2>(iter, lcm_string);\n });\n #else\n AT_DISPATCH_INTEGRAL_TYPES(iter.common_dtype(), \"lcm_hip\", [&]() {\n gpu_kernel(iter, [] GPU_LAMBDA (scalar_t a, scalar_t b) -> scalar_t {\n scalar_t g = calc_gcd(a, b);\n return (g == 0) ? 0 : ::abs(a / g * b);\n });\n });\n #endif // AT_USE_JITERATOR()\n}\n\nREGISTER_DISPATCH(gcd_stub, &gcd_kernel_hip);\nREGISTER_DISPATCH(lcm_stub, &lcm_kernel_hip);\n\n} // namespace at::native\n###"1173 },1174 {1175 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char hermite_polynomial_h_name[] = \"hermite_polynomial_h_forward\";\n\n void hermite_polynomial_h_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_h_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<hermite_polynomial_h_name, scalar_t, scalar_t>(iterator, hermite_polynomial_h_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_h_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return hermite_polynomial_h_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // hermite_polynomial_h_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(hermite_polynomial_h_stub, &hermite_polynomial_h_kernel_cuda);\n} // namespace at::native\n\n\n###",1176 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char hermite_polynomial_h_name[] = \"hermite_polynomial_h_forward\";\n\n void hermite_polynomial_h_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_h_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<hermite_polynomial_h_name, scalar_t, scalar_t>(iterator, hermite_polynomial_h_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_h_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return hermite_polynomial_h_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // hermite_polynomial_h_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(hermite_polynomial_h_stub, &hermite_polynomial_h_kernel_hip);\n} // namespace at::native\n###"1177 },1178 {1179 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char hermite_polynomial_he_name[] = \"hermite_polynomial_he_forward\";\n\n void hermite_polynomial_he_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_he_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<hermite_polynomial_he_name, scalar_t, scalar_t>(iterator, hermite_polynomial_he_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_he_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return hermite_polynomial_he_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // hermite_polynomial_he_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(hermite_polynomial_he_stub, &hermite_polynomial_he_kernel_cuda);\n} // namespace at::native\n\n\n###",1180 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char hermite_polynomial_he_name[] = \"hermite_polynomial_he_forward\";\n\n void hermite_polynomial_he_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_he_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<hermite_polynomial_he_name, scalar_t, scalar_t>(iterator, hermite_polynomial_he_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"hermite_polynomial_he_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return hermite_polynomial_he_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // hermite_polynomial_he_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(hermite_polynomial_he_stub, &hermite_polynomial_he_kernel_hip);\n} // namespace at::native\n###"1181 },1182 {1183 "cuda": "\n#pragma once\n\n#include <ATen/ceil_div.h>\n#include <ATen/cuda/DeviceUtils.cuh>\n#include <ATen/cuda/AsmUtils.cuh>\n#include <c10/macros/Macros.h>\n\n// Collection of in-kernel scan / prefix sum utilities\n\nnamespace at {\nnamespace cuda {\n\n// Inclusive prefix sum for binary vars using intra-warp voting +\n// shared memory\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void inclusiveBinaryPrefixScan(T* smem, bool in, T* out, BinaryFunction binop) {\n // Within-warp, we use warp voting.\n#if defined (USE_ROCM)\n unsigned long long int vote = WARP_BALLOT(in);\n T index = __popcll(getLaneMaskLe() & vote);\n T carry = __popcll(vote);\n#else\n T vote = WARP_BALLOT(in);\n T index = __popc(getLaneMaskLe() & vote);\n T carry = __popc(vote);\n#endif\n\n int warp = threadIdx.x / C10_WARP_SIZE;\n\n // Per each warp, write out a value\n if (getLaneId() == 0) {\n smem[warp] = carry;\n }\n\n __syncthreads();\n\n // Sum across warps in one thread. This appears to be faster than a\n // warp shuffle scan for CC 3.0+\n if (threadIdx.x == 0) {\n int current = 0;\n for (int i = 0; i < blockDim.x / C10_WARP_SIZE; ++i) {\n T v = smem[i];\n smem[i] = binop(smem[i], current);\n current = binop(current, v);\n }\n }\n\n __syncthreads();\n\n // load the carry from the preceding warp\n if (warp >= 1) {\n index = binop(index, smem[warp - 1]);\n }\n\n *out = index;\n\n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n// Exclusive prefix sum for binary vars using intra-warp voting +\n// shared memory\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void exclusiveBinaryPrefixScan(T* smem, bool in, T* out, T* carry, BinaryFunction binop) {\n inclusiveBinaryPrefixScan<T, false, BinaryFunction>(smem, in, out, binop);\n\n // Inclusive to exclusive\n *out -= (T) in;\n\n // The outgoing carry for all threads is the last warp's sum\n *carry = smem[at::ceil_div<int>(blockDim.x, C10_WARP_SIZE) - 1];\n\n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n}} // namespace at::cuda\n\n\n###",1184 "hip": " // !!! This is a file automatically generated by hipify!!!\n#include \"hip/hip_runtime.h\"\n#pragma once\n\n#include <ATen/ceil_div.h>\n#include <ATen/hip\\DeviceUtils.cuh>\n#include <ATen/hip\\AsmUtils.cuh>\n#include <c10/macros/Macros.h>\n\n// Collection of in-kernel scan / prefix sum utilities\n\nnamespace at {\nnamespace hip {\n\n// Inclusive prefix sum for binary vars using intra-warp voting +\n// shared memory\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void inclusiveBinaryPrefixScan(T* smem, bool in, T* out, BinaryFunction binop) {\n // Within-warp, we use warp voting.\n#if defined (USE_ROCM)\n unsigned long long int vote = WARP_BALLOT(in);\n T index = __popcll(getLaneMaskLe() & vote);\n T carry = __popcll(vote);\n#else\n T vote = WARP_BALLOT(in);\n T index = __popc(getLaneMaskLe() & vote);\n T carry = __popc(vote);\n#endif\n\n int warp = threadIdx.x / C10_WARP_SIZE;\n\n // Per each warp, write out a value\n if (getLaneId() == 0) {\n smem[warp] = carry;\n }\n\n __syncthreads();\n\n // Sum across warps in one thread. This appears to be faster than a\n // warp shuffle scan for CC 3.0+\n if (threadIdx.x == 0) {\n int current = 0;\n for (int i = 0; i < blockDim.x / C10_WARP_SIZE; ++i) {\n T v = smem[i];\n smem[i] = binop(smem[i], current);\n current = binop(current, v);\n }\n }\n\n __syncthreads();\n\n // load the carry from the preceding warp\n if (warp >= 1) {\n index = binop(index, smem[warp - 1]);\n }\n\n *out = index;\n\n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n// Exclusive prefix sum for binary vars using intra-warp voting +\n// shared memory\ntemplate <typename T, bool KillWARDependency, class BinaryFunction>\n__device__ void exclusiveBinaryPrefixScan(T* smem, bool in, T* out, T* carry, BinaryFunction binop) {\n inclusiveBinaryPrefixScan<T, false, BinaryFunction>(smem, in, out, binop);\n\n // Inclusive to exclusive\n *out -= (T) in;\n\n // The outgoing carry for all threads is the last warp's sum\n *carry = smem[at::ceil_div<int>(blockDim.x, C10_WARP_SIZE) - 1];\n\n if (KillWARDependency) {\n __syncthreads();\n }\n}\n\n}} // namespace at::cuda\n###"1185 },1186 {1187 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char laguerre_polynomial_l_name[] = \"laguerre_polynomial_l_forward\";\n\n void laguerre_polynomial_l_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"laguerre_polynomial_l_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<laguerre_polynomial_l_name, scalar_t, scalar_t>(iterator, laguerre_polynomial_l_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"laguerre_polynomial_l_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return laguerre_polynomial_l_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // laguerre_polynomial_l_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(laguerre_polynomial_l_stub, &laguerre_polynomial_l_kernel_cuda);\n} // namespace at::native\n\n\n###",1188 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char laguerre_polynomial_l_name[] = \"laguerre_polynomial_l_forward\";\n\n void laguerre_polynomial_l_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"laguerre_polynomial_l_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<laguerre_polynomial_l_name, scalar_t, scalar_t>(iterator, laguerre_polynomial_l_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"laguerre_polynomial_l_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return laguerre_polynomial_l_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // laguerre_polynomial_l_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(laguerre_polynomial_l_stub, &laguerre_polynomial_l_kernel_hip);\n} // namespace at::native\n###"1189 },1190 {1191 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n\nnamespace at::native {\n namespace {\n const char legendre_polynomial_p_name[] = \"legendre_polynomial_p_forward\";\n\n void legendre_polynomial_p_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"legendre_polynomial_p_cuda\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<legendre_polynomial_p_name, scalar_t, scalar_t>(iterator, legendre_polynomial_p_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"legendre_polynomial_p_cuda\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return legendre_polynomial_p_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // legendre_polynomial_p_kernel_cuda\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(legendre_polynomial_p_stub, &legendre_polynomial_p_kernel_cuda);\n} // namespace at::native\n\n\n###",1192 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/Dispatch.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n\nnamespace at::native {\n namespace {\n const char legendre_polynomial_p_name[] = \"legendre_polynomial_p_forward\";\n\n void legendre_polynomial_p_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"legendre_polynomial_p_hip\", [&]() {\n opmath_jitted_gpu_kernel_with_scalars<legendre_polynomial_p_name, scalar_t, scalar_t>(iterator, legendre_polynomial_p_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"legendre_polynomial_p_hip\", [&]() {\n gpu_kernel_with_scalars(iterator, []GPU_LAMBDA(scalar_t x, scalar_t n) -> scalar_t {\n return legendre_polynomial_p_forward<scalar_t, true>(x, n);\n });\n });\n#endif\n } // legendre_polynomial_p_kernel_hip\n } // namespace (anonymous)\n\n REGISTER_DISPATCH(legendre_polynomial_p_stub, &legendre_polynomial_p_kernel_hip);\n} // namespace at::native\n###"1193 },1194 {1195 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/OpMathType.h>\n#include <c10/util/MathConstants.h>\n\n// NOTE: CUDA on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid logaddexp_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::BFloat16, ScalarType::Half,\n iter.dtype(), \"logaddexp_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n gpu_kernel(iter, [] GPU_LAMBDA (scalar_t a_, scalar_t b_) -> scalar_t {\n const auto a = static_cast<opmath_t>(a_);\n const auto b = static_cast<opmath_t>(b_);\n if (::isinf(a) && a == b) {\n return a;\n } else {\n const auto m = ::max(a, b);\n return m + ::log1p(::exp(-::abs(a - b)));\n }\n });\n });\n}\n\nvoid logaddexp2_kernel_cuda(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND(\n ScalarType::BFloat16,\n iter.dtype(), \"logaddexp2_cuda\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const auto inv_log_2 = static_cast<opmath_t>(1.0 / c10::ln_2<double>);\n gpu_kernel(iter, [inv_log_2] GPU_LAMBDA (scalar_t a_, scalar_t b_) -> scalar_t {\n const auto a = static_cast<opmath_t>(a_);\n const auto b = static_cast<opmath_t>(b_);\n if (::isinf(a) && a == b) {\n return a;\n } else {\n const auto m = ::max(a, b);\n return m + ::log1p(::exp2(-::abs(a - b))) * inv_log_2;\n }\n });\n });\n}\n\nREGISTER_DISPATCH(logaddexp_stub, &logaddexp_kernel_cuda);\nREGISTER_DISPATCH(logaddexp2_stub, &logaddexp2_kernel_cuda);\n\n} // namespace at::native\n\n\n###",1196 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/BinaryOps.h>\n#include <ATen/OpMathType.h>\n#include <c10/util/MathConstants.h>\n\n// NOTE: HIP on Windows requires that the enclosing function\n// of a __device__ lambda not have internal linkage.\n\nnamespace at::native {\n\nvoid logaddexp_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND2(\n ScalarType::BFloat16, ScalarType::Half,\n iter.dtype(), \"logaddexp_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n gpu_kernel(iter, [] GPU_LAMBDA (scalar_t a_, scalar_t b_) -> scalar_t {\n const auto a = static_cast<opmath_t>(a_);\n const auto b = static_cast<opmath_t>(b_);\n if (::isinf(a) && a == b) {\n return a;\n } else {\n const auto m = ::max(a, b);\n return m + ::log1p(::exp(-::abs(a - b)));\n }\n });\n });\n}\n\nvoid logaddexp2_kernel_hip(TensorIteratorBase& iter) {\n AT_DISPATCH_FLOATING_TYPES_AND(\n ScalarType::BFloat16,\n iter.dtype(), \"logaddexp2_hip\",\n [&]() {\n using opmath_t = at::opmath_type<scalar_t>;\n const auto inv_log_2 = static_cast<opmath_t>(1.0 / c10::ln_2<double>);\n gpu_kernel(iter, [inv_log_2] GPU_LAMBDA (scalar_t a_, scalar_t b_) -> scalar_t {\n const auto a = static_cast<opmath_t>(a_);\n const auto b = static_cast<opmath_t>(b_);\n if (::isinf(a) && a == b) {\n return a;\n } else {\n const auto m = ::max(a, b);\n return m + ::log1p(::exp2(-::abs(a - b))) * inv_log_2;\n }\n });\n });\n}\n\nREGISTER_DISPATCH(logaddexp_stub, &logaddexp_kernel_hip);\nREGISTER_DISPATCH(logaddexp2_stub, &logaddexp2_kernel_hip);\n\n} // namespace at::native\n###"1197 },1198 {1199 "cuda": "\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/cuda/JitLoops.cuh>\n#include <ATen/native/cuda/Loops.cuh>\n#include <ATen/native/cuda/Math.cuh>\n#include <ATen/native/cuda/jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/cuda/CUDAMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_CUDA char modified_bessel_i0_name[] = \"modified_bessel_i0_forward\";\n\n void modified_bessel_i0_kernel_cuda(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"modified_bessel_i0_cuda\", [&]() {\n jitted_gpu_kernel<modified_bessel_i0_name, scalar_t, scalar_t, 1>(iterator, modified_bessel_i0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"modified_bessel_i0_cuda\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return modified_bessel_i0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_modified_bessel_i0_stub, &modified_bessel_i0_kernel_cuda);\n} // namespace at::native\n\n\n###",1200 "hip": " // !!! This is a file automatically generated by hipify!!!\n#define TORCH_ASSERT_NO_OPERATORS\n\n#include <ATen/native/UnaryOps.h>\n\n#include <limits>\n\n#include <ATen/AccumulateType.h>\n#include <ATen/Dispatch.h>\n#include <ATen/native/DispatchStub.h>\n#include <ATen/native/Math.h>\n#include <ATen/native/TensorIterator.h>\n#include <ATen/native/hip\\JitLoops.cuh>\n#include <ATen/native/hip\\Loops.cuh>\n#include <ATen/native/hip\\Math.cuh>\n#include <ATen/native/hip\\jit_utils.h>\n#include <ATen/NumericUtils.h>\n#include <c10/core/Scalar.h>\n#include <c10/hip/HIPMathCompat.h>\n#include <c10/util/complex.h>\n\nnamespace at::native {\n namespace {\n CONSTEXPR_EXCEPT_WIN_HIP char modified_bessel_i0_name[] = \"modified_bessel_i0_forward\";\n\n void modified_bessel_i0_kernel_hip(TensorIteratorBase& iterator) {\n#if AT_USE_JITERATOR()\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"modified_bessel_i0_hip\", [&]() {\n jitted_gpu_kernel<modified_bessel_i0_name, scalar_t, scalar_t, 1>(iterator, modified_bessel_i0_string);\n });\n#else\n AT_DISPATCH_FLOATING_TYPES(iterator.common_dtype(), \"modified_bessel_i0_hip\", [&]() {\n gpu_kernel(iterator, []GPU_LAMBDA(scalar_t a) -> scalar_t {\n return modified_bessel_i0_forward(a);\n });\n });\n#endif // AT_USE_JITERATOR()\n }\n }\n\n REGISTER_DISPATCH(special_modified_bessel_i0_stub, &modified_bessel_i0_kernel_hip);\n} // namespace at::native\n###"