echodada/Qwen-7B

1#include <torch/all.h>2#include <torch/python.h>3#include <c10/cuda/CUDAGuard.h>4 5// adapted from https://github.com/PanQiWei/AutoGPTQ/blob/main/autogptq_extension/cuda_256/autogptq_cuda_256.cpp6void vecquant8matmul_cuda(7  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,8  torch::Tensor scales, torch::Tensor zeros,9  torch::Tensor g_idx10);11 12void vecquant8matmul(13  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,14  torch::Tensor scales, torch::Tensor zeros,15  torch::Tensor g_idx16) {17  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));18  vecquant8matmul_cuda(vec, mat, mul, scales, zeros, g_idx);19}20 21void vecquant8matmul_batched_cuda(22  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,23  torch::Tensor scales, torch::Tensor zeros24);25 26void vecquant8matmul_batched(27  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,28  torch::Tensor scales, torch::Tensor zeros29) {30  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));31  vecquant8matmul_batched_cuda(vec, mat, mul, scales, zeros);32}33 34void vecquant8matmul_batched_column_compression_cuda(35  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,36  torch::Tensor scales, torch::Tensor zeros37);38 39void vecquant8matmul_batched_column_compression(40  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,41  torch::Tensor scales, torch::Tensor zeros42) {43  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));44  vecquant8matmul_batched_column_compression_cuda(vec, mat, mul, scales, zeros);45}46 47void vecquant4matmul_batched_cuda(48  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,49  torch::Tensor scales, torch::Tensor zeros50);51 52void vecquant4matmul_batched(53  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,54  torch::Tensor scales, torch::Tensor zeros55) {56  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));57  vecquant4matmul_batched_cuda(vec, mat, mul, scales, zeros);58}59 60void vecquant4matmul_batched_column_compression_cuda(61  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,62  torch::Tensor scales, torch::Tensor zeros63);64 65void vecquant4matmul_batched_column_compression(66  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,67  torch::Tensor scales, torch::Tensor zeros68) {69  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));70  vecquant4matmul_batched_column_compression_cuda(vec, mat, mul, scales, zeros);71}72 73void vecquant8matmul_batched_old_cuda(74  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,75  torch::Tensor scales, torch::Tensor zeros76);77 78void vecquant8matmul_batched_old(79  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,80  torch::Tensor scales, torch::Tensor zeros81) {82  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));83  vecquant8matmul_batched_old_cuda(vec, mat, mul, scales, zeros);84}85 86 87void vecquant4matmul_batched_old_cuda(88  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,89  torch::Tensor scales, torch::Tensor zeros90);91 92void vecquant4matmul_batched_old(93  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,94  torch::Tensor scales, torch::Tensor zeros95) {96  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));97  vecquant4matmul_batched_old_cuda(vec, mat, mul, scales, zeros);98}99 100void vecquant8matmul_batched_column_compression_old_cuda(101  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,102  torch::Tensor scales, torch::Tensor zeros103);104 105void vecquant8matmul_batched_column_compression_old(106  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,107  torch::Tensor scales, torch::Tensor zeros108) {109  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));110  vecquant8matmul_batched_column_compression_old_cuda(vec, mat, mul, scales, zeros);111}112 113void vecquant4matmul_batched_column_compression_old_cuda(114  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,115  torch::Tensor scales, torch::Tensor zeros116);117 118void vecquant4matmul_batched_column_compression_old(119  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,120  torch::Tensor scales, torch::Tensor zeros121) {122  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));123  vecquant4matmul_batched_column_compression_old_cuda(vec, mat, mul, scales, zeros);124}125 126 127 128void vecquant8matmul_batched_faster_cuda(129  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,130  torch::Tensor scales, torch::Tensor zeros131);132 133void vecquant8matmul_batched_faster(134  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,135  torch::Tensor scales, torch::Tensor zeros136) {137  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));138  vecquant8matmul_batched_faster_cuda(vec, mat, mul, scales, zeros);139}140 141 142void vecquant8matmul_batched_faster_old_cuda(143  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,144  torch::Tensor scales, torch::Tensor zeros145);146 147void vecquant8matmul_batched_faster_old(148  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,149  torch::Tensor scales, torch::Tensor zeros150) {151  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));152  vecquant8matmul_batched_faster_old_cuda(vec, mat, mul, scales, zeros);153}154 155void vecquant8matmul_batched_column_compression_faster_cuda(156  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,157  torch::Tensor scales, torch::Tensor zeros158);159 160void vecquant8matmul_batched_column_compression_faster(161  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,162  torch::Tensor scales, torch::Tensor zeros163) {164  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));165  vecquant8matmul_batched_column_compression_faster_cuda(vec, mat, mul, scales, zeros);166}167 168 169void vecquant8matmul_batched_column_compression_faster_old_cuda(170  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,171  torch::Tensor scales, torch::Tensor zeros172);173 174void vecquant8matmul_batched_column_compression_faster_old(175  torch::Tensor vec, torch::Tensor mat, torch::Tensor mul,176  torch::Tensor scales, torch::Tensor zeros177) {178  const at::cuda::OptionalCUDAGuard device_guard(device_of(vec));179  vecquant8matmul_batched_column_compression_faster_old_cuda(vec, mat, mul, scales, zeros);180}181 182 183 184PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {185  m.def("vecquant8matmul", &vecquant8matmul, "Vector 8-bit Quantized Matrix Multiplication (CUDA) (desc_act)");186  m.def("vecquant8matmul_batched", &vecquant8matmul_batched, "Vector 8-bit Batched Quantized Matrix Multiplication (CUDA) (desc_act)");187  m.def("vecquant8matmul_batched_old", &vecquant8matmul_batched_old, "Vector 8-bit old Batched Quantized Matrix Multiplication (CUDA) (desc_act)");188  m.def("vecquant8matmul_batched_faster", &vecquant8matmul_batched_faster, "Vector 8-bit old Batched Quantized Matrix Multiplication (CUDA) (desc_act)");189  m.def("vecquant8matmul_batched_faster_old", &vecquant8matmul_batched_faster_old, "Vector 8-bit old Batched Quantized Matrix Multiplication (CUDA) (desc_act)");190  m.def("vecquant4matmul_batched_old", &vecquant4matmul_batched_old, "Vector 4-bit old Batched Quantized Matrix Multiplication (CUDA) (desc_act)");191  m.def("vecquant8matmul_batched_column_compression", &vecquant8matmul_batched_column_compression, "Vector 8-bit Batched Quantized Matrix Multiplication (CUDA) with weight's column compressed (desc_act)");192  m.def("vecquant8matmul_batched_column_compression_old", &vecquant8matmul_batched_column_compression_old, "Vector old 8-bit Batched Quantized Matrix Multiplication (CUDA) with weight's column compressed (desc_act)");193  m.def("vecquant8matmul_batched_column_compression_faster", &vecquant8matmul_batched_column_compression_faster, "Vector old 8-bit Batched Quantized Matrix Multiplication (CUDA) with weight's column compressed (desc_act)");194  m.def("vecquant8matmul_batched_column_compression_faster_old", &vecquant8matmul_batched_column_compression_faster_old, "Vector old 8-bit Batched Quantized Matrix Multiplication (CUDA) with weight's column compressed (desc_act)");195  m.def("vecquant4matmul_batched_column_compression_old", &vecquant4matmul_batched_column_compression_old, "Vector old 4-bit Batched Quantized Matrix Multiplication (CUDA) with weight's column compressed (desc_act)");196  m.def("vecquant4matmul_batched", &vecquant4matmul_batched, "Vector 4-bit Batched Quantized Matrix Multiplication (CUDA) (desc_act)");197  m.def("vecquant4matmul_batched_column_compression", &vecquant4matmul_batched_column_compression, "Vector 4-bit Batched Quantized Matrix Multiplication (CUDA) with weight's column compressed (desc_act)");198}199