diff --git a/S1/hli28146_#99/PGELU_cuda.py b/S1/hli28146_#99/PGELU_cuda.py new file mode 100644 index 00000000..f0d28d3c --- /dev/null +++ b/S1/hli28146_#99/PGELU_cuda.py @@ -0,0 +1,117 @@ +import torch +import torch.nn as nn +from torch.utils.cpp_extension import load_inline +import math + +cpp_source = """ +#include + +torch::Tensor pgelu_cuda_forward( + const torch::Tensor& input, + const torch::Tensor& alpha, + const torch::Tensor& beta); +""" + +cuda_source = """ +#include +#include +#include + +#define BLOCK_SIZE 256 +#define PI_HALF 1.5707963267948966 +#define EPS 1e-7 + +// double2 for 128-bit vectorization +struct __align__(16) Double2 { + double x, y; +}; + +// P-GELU for double +__device__ __forceinline__ double compute_pgelu_double(double x, double alpha, double beta) { + double inner = alpha * x + beta * x * x * x; + double clamped_inner = fmin(fmax(inner, -PI_HALF + EPS), PI_HALF - EPS); + return x * (1.0 + tan(clamped_inner)); +} + +template +__global__ void pgelu_kernel( + T* __restrict__ output, + const T* __restrict__ input, + const int n, + T alpha, + T beta) +{ + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + const int vec_n = n / 2; // double2 + + int i = idx; + const int stride = blockDim.x * gridDim.x; + + for (; i < vec_n; i += stride) { + Double2 in_vec = reinterpret_cast(input)[i]; + Double2 out_vec; + + out_vec.x = compute_pgelu_double(in_vec.x, alpha, beta); + out_vec.y = compute_pgelu_double(in_vec.y, alpha, beta); + + reinterpret_cast(output)[i] = out_vec; + } + + int tail_idx = vec_n * 2 + idx; + if (idx == 0 && tail_idx < n) { + output[tail_idx] = compute_pgelu_double(input[tail_idx], alpha, beta); + } +} + +torch::Tensor pgelu_cuda_forward( + const torch::Tensor& input, + const torch::Tensor& alpha_t, + const torch::Tensor& beta_t) +{ + TORCH_CHECK(input.is_cuda(), "Input must be a CUDA tensor"); + TORCH_CHECK(input.is_contiguous(), "Input must be contiguous"); + + const int n = input.numel(); + auto output = torch::empty_like(input); + + const double alpha = alpha_t.item(); + const double beta = beta_t.item(); + + const int vec_n = n / 2; + const int grid_size = (vec_n + BLOCK_SIZE - 1) / BLOCK_SIZE; + + int final_grid = (grid_size < 1) ? 1 : grid_size; + if (final_grid > 65535) final_grid = 65535; + + AT_DISPATCH_FLOATING_TYPES(input.scalar_type(), "pgelu_kernel", ([&]{ + pgelu_kernel<<>>( + output.data_ptr(), + input.data_ptr(), + n, + static_cast(alpha), + static_cast(beta) + ); + })); + + return output; +} +""" + +pgelu_op_module = load_inline( + name='pgelu_tan_op_double', + cpp_sources=cpp_source, + cuda_sources=cuda_source, + functions=['pgelu_cuda_forward'], + verbose=False, + extra_cuda_cflags=['-O3'] +) + +class ModelNew(nn.Module): + def __init__(self, alpha_init=1.0, beta_init=1.0): + super(ModelNew, self).__init__() + self.alpha = nn.Parameter(torch.tensor(alpha_init, dtype=torch.float64)) + self.beta = nn.Parameter(torch.tensor(beta_init, dtype=torch.float64)) + self.op = pgelu_op_module + + def forward(self, input_tensor: torch.Tensor) -> torch.Tensor: + return self.op.pgelu_cuda_forward(input_tensor.contiguous(), self.alpha, self.beta) \ No newline at end of file diff --git a/S1/hli28146_#99/PGELU_torch.py b/S1/hli28146_#99/PGELU_torch.py new file mode 100644 index 00000000..df9902a0 --- /dev/null +++ b/S1/hli28146_#99/PGELU_torch.py @@ -0,0 +1,45 @@ +import torch +import torch.nn as nn +import math + +BATCH_SIZE = 4096 +HIDDEN_DIM = 4096 +SHAPE = (BATCH_SIZE, HIDDEN_DIM) + +ALPHA_INIT = 1.0 +BETA_INIT = 1.0 + +DTYPE = torch.float64 + +class PGELU(nn.Module): + ''' + P-GELU: A Novel Activation Function to Optimize Whisper for Darija Speech Translation + https://ieeexplore.ieee.org/document/11016691 + Formula: f(x) = x * (1 + tan(alpha * x + beta * x^3)) + ''' + def __init__(self, alpha_init=1.0, beta_init=1.0): + super(PGELU, self).__init__() + self.alpha = nn.Parameter(torch.tensor(alpha_init, dtype=DTYPE)) + self.beta = nn.Parameter(torch.tensor(beta_init, dtype=DTYPE)) + self.pi_half = math.pi / 2.0 + self.eps = 1e-7 # Epsilon for double + + def forward(self, x: torch.Tensor) -> torch.Tensor: + inner = self.alpha * x + self.beta * x.pow(3) + inner_clamped = torch.clamp(inner, -self.pi_half + self.eps, self.pi_half - self.eps) + return x * (1.0 + torch.tan(inner_clamped)) + +class Model(nn.Module): + def __init__(self, alpha_init=1.0, beta_init=1.0): + super(Model, self).__init__() + self.act = PGELU(alpha_init, beta_init) + + def forward(self, x): + return self.act(x) + +def get_inputs(): + input_tensor = torch.randn(SHAPE, dtype=DTYPE) + return [input_tensor.contiguous()] + +def get_init_inputs(): + return [ALPHA_INIT, BETA_INIT] \ No newline at end of file diff --git a/S1/hli28146_#99/prompt.txt b/S1/hli28146_#99/prompt.txt new file mode 100644 index 00000000..1050bbde --- /dev/null +++ b/S1/hli28146_#99/prompt.txt @@ -0,0 +1,71 @@ +Write a custom CUDA kernel to optimize `P-GELU` using `float64` (double) precision. + +Formula: f(x) = x * (1 + tan(alpha * x + beta * x^3)) + +Problem Analysis: +1. Precision Issues with float32: The combination of a cubic polynomial and the `tan` function amplifies floating-point rounding errors. +2. Memory Bottleneck: The operation is memory-bound, now with 8 bytes per element. +3. Numerical Stability: The input to `tan` must be clamped to avoid asymptotes. + +Optimization Strategy: Fused Element-wise Kernel with Double Precision + +1. Data Type: All computations are performed in `double`. + +2. Vectorized Loads (double2): Use `double2` to load 128 bits (2 double elements) per memory transaction. + +3. Fused Stable Math (in double): + - For `x`, compute `inner = alpha * x + beta * x*x*x`. + - Clamp `inner` to stay away from `pi/2`. + - Compute `result = x * (1.0 + tan(inner))`. + - Use standard `double` precision math functions (`tan`). + +4. One-Pass: Fuse all logic into a single read-compute-write kernel. + +Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is: + +```python +import torch +import torch.nn as nn +import math + +BATCH_SIZE = 4096 +HIDDEN_DIM = 4096 +SHAPE = (BATCH_SIZE, HIDDEN_DIM) + +ALPHA_INIT = 1.0 +BETA_INIT = 1.0 + +DTYPE = torch.float64 + +class PGELU(nn.Module): + ''' + P-GELU: A Novel Activation Function to Optimize Whisper for Darija Speech Translation + https://ieeexplore.ieee.org/document/11016691 + Formula: f(x) = x * (1 + tan(alpha * x + beta * x^3)) + ''' + def __init__(self, alpha_init=1.0, beta_init=1.0): + super(PGELU, self).__init__() + self.alpha = nn.Parameter(torch.tensor(alpha_init, dtype=DTYPE)) + self.beta = nn.Parameter(torch.tensor(beta_init, dtype=DTYPE)) + self.pi_half = math.pi / 2.0 + self.eps = 1e-7 # Epsilon for double + + def forward(self, x: torch.Tensor) -> torch.Tensor: + inner = self.alpha * x + self.beta * x.pow(3) + inner_clamped = torch.clamp(inner, -self.pi_half + self.eps, self.pi_half - self.eps) + return x * (1.0 + torch.tan(inner_clamped)) + +class Model(nn.Module): + def __init__(self, alpha_init=1.0, beta_init=1.0): + super(Model, self).__init__() + self.act = PGELU(alpha_init, beta_init) + + def forward(self, x): + return self.act(x) + +def get_inputs(): + input_tensor = torch.randn(SHAPE, dtype=DTYPE) + return [input_tensor.contiguous()] + +def get_init_inputs(): + return [ALPHA_INIT, BETA_INIT] \ No newline at end of file diff --git a/S1/hli28146_#99/run_code.py b/S1/hli28146_#99/run_code.py new file mode 100644 index 00000000..6e754ddf --- /dev/null +++ b/S1/hli28146_#99/run_code.py @@ -0,0 +1,74 @@ +########################################################### +# 性能和精度验证程序 +########################################################### +import torch +import torch.nn as nn +import time +from PGELU_torch import Model,get_inputs,get_init_inputs +from PGELU_cuda import ModelNew + +def run_benchmark(): + # 检查 CUDA 是否可用 + if not torch.cuda.is_available(): + print("CUDA 不可用,请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。") + return + else: + device = torch.device("cuda") + + # 初始化模型 + init_inputs = get_init_inputs() + init_inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs + ] + inputs = get_inputs() + inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs + ] + + torch_model = Model(*init_inputs).cuda() + cuda_model = ModelNew(*init_inputs).cuda() + + torch_model.eval() + cuda_model.eval() + + print("-------------------- 精度对齐验证 --------------------") + with torch.no_grad(): + output_torch = torch_model( *inputs) + output_cuda = cuda_model(*inputs) + + precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03) + if precision_flag: + print("✅ 精度对齐:两个模型的输出结果非常接近。") + else: + print("❌ 精度不一致!") + + print("\n-------------------- 性能加速比测试 --------------------") + num_iterations = 100 + + # PyTorch 模型计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = torch_model(*inputs) + torch.cuda.synchronize() + torch_time = (time.time() - start_time) / num_iterations + + # 自定义 CUDA 内核计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = cuda_model(*inputs) + torch.cuda.synchronize() + cuda_time = (time.time() - start_time) / num_iterations + + print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒") + print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒") + speedup = 0 + if cuda_time > 0: + speedup = torch_time / cuda_time + print(f"加速比 (Speedup): {speedup:.2f}x") + else: + print("CUDA 内核执行时间为0,无法计算加速比。") + return precision_flag,speedup +if __name__ == "__main__": + precision_flag,speedup = run_benchmark() \ No newline at end of file