From 613b7d7c622f868dda5cdc4eeef7428d2fff92e5 Mon Sep 17 00:00:00 2001 From: hli28146 Date: Wed, 10 Dec 2025 19:39:47 +0800 Subject: [PATCH] finish Sb-PiPLU #116 --- S1/hli28146_#116/SbPiPLU_cuda.py | 115 ++++++++++++++++++++++++++++++ S1/hli28146_#116/SbPiPLU_torch.py | 50 +++++++++++++ S1/hli28146_#116/prompt.txt | 78 ++++++++++++++++++++ S1/hli28146_#116/run_code.py | 74 +++++++++++++++++++ 4 files changed, 317 insertions(+) create mode 100644 S1/hli28146_#116/SbPiPLU_cuda.py create mode 100644 S1/hli28146_#116/SbPiPLU_torch.py create mode 100644 S1/hli28146_#116/prompt.txt create mode 100644 S1/hli28146_#116/run_code.py diff --git a/S1/hli28146_#116/SbPiPLU_cuda.py b/S1/hli28146_#116/SbPiPLU_cuda.py new file mode 100644 index 00000000..8629bbd0 --- /dev/null +++ b/S1/hli28146_#116/SbPiPLU_cuda.py @@ -0,0 +1,115 @@ +import torch +import torch.nn as nn +from torch.utils.cpp_extension import load_inline + +cpp_source = """ +#include + +torch::Tensor sb_piplu_cuda_forward(const torch::Tensor& input, const torch::Tensor& k_tensor); +""" + +cuda_source = """ +#include +#include +#include + +#define BLOCK_SIZE 256 + +struct __align__(16) Float4 { + float x, y, z, w; +}; + +// Sb-PiPLU Logic +__device__ __forceinline__ float compute_sb_piplu(float x, float k, float inv_k) { + if (x > k) { + return x * inv_k; + } else if (x > 0.0f) { // 0 < x <= k + return x; + } else { // x <= 0 + float ss = x / (1.0f + fabsf(x)); + return 2.0f * ss + ss * ss; + } +} + +__global__ void sb_piplu_kernel( + float* __restrict__ output, + const float* __restrict__ input, + const int n, + const float k, + const float inv_k) +{ + const int idx = blockIdx.x * blockDim.x + threadIdx.x; + const int vec_n = n / 4; + + int i = idx; + const int stride = blockDim.x * gridDim.x; + + for (; i < vec_n; i += stride) { + Float4 in_vec = reinterpret_cast(input)[i]; + Float4 out_vec; + + out_vec.x = compute_sb_piplu(in_vec.x, k, inv_k); + out_vec.y = compute_sb_piplu(in_vec.y, k, inv_k); + out_vec.z = compute_sb_piplu(in_vec.z, k, inv_k); + out_vec.w = compute_sb_piplu(in_vec.w, k, inv_k); + + reinterpret_cast(output)[i] = out_vec; + } + + int start_scalar = vec_n * 4; + int global_tid = blockIdx.x * blockDim.x + threadIdx.x; + int total_threads = gridDim.x * gridDim.x; + + int current_idx = start_scalar + global_tid; + while (current_idx < n) { + output[current_idx] = compute_sb_piplu(input[current_idx], k, inv_k); + current_idx += total_threads; + } +} + +torch::Tensor sb_piplu_cuda_forward(const torch::Tensor& input, const torch::Tensor& k_tensor) { + TORCH_CHECK(input.is_cuda(), "Input must be a CUDA tensor"); + TORCH_CHECK(input.is_contiguous(), "Input must be contiguous"); + + const int n = input.numel(); + auto output = torch::empty_like(input); + + // Extract scalar value from k_tensor on the host + const float k = k_tensor.item(); + const float inv_k = 1.0f / k; + + const int vec_n = n / 4; + const int grid_size = (vec_n + BLOCK_SIZE - 1) / BLOCK_SIZE; + + int final_grid = (grid_size < 1) ? 1 : grid_size; + if (final_grid > 65535) final_grid = 65535; + + sb_piplu_kernel<<>>( + output.data_ptr(), + input.data_ptr(), + n, + k, + inv_k + ); + + return output; +} +""" + +sb_piplu_op_module = load_inline( + name='sb_piplu_param_op', + cpp_sources=cpp_source, + cuda_sources=cuda_source, + functions=['sb_piplu_cuda_forward'], + verbose=False, + extra_cuda_cflags=['-O3'] +) + +class ModelNew(nn.Module): + def __init__(self, k_init=21.0): + super(ModelNew, self).__init__() + self.k = nn.Parameter(torch.tensor(k_init)) + self.op = sb_piplu_op_module + + def forward(self, input_tensor: torch.Tensor) -> torch.Tensor: + return self.op.sb_piplu_cuda_forward(input_tensor.contiguous(), self.k) \ No newline at end of file diff --git a/S1/hli28146_#116/SbPiPLU_torch.py b/S1/hli28146_#116/SbPiPLU_torch.py new file mode 100644 index 00000000..43659934 --- /dev/null +++ b/S1/hli28146_#116/SbPiPLU_torch.py @@ -0,0 +1,50 @@ +import torch +import torch.nn as nn +import torch.nn.functional as F + +BATCH_SIZE = 4096 +HIDDEN_DIM = 4096 +SHAPE = (BATCH_SIZE, HIDDEN_DIM) + +# Sb-PiPLU 初始参数 k +K_INIT = 21.0 + +class SbPiPLU(nn.Module): + ''' + Sb-PiPLU: A Novel Parametric Activation Function for Deep Learning + DOI:10.1109/ACCESS.2025.3561464 + Formula: + f(x) = 2*softsign(x) + softsign(x)^2 if x <= 0 + = x if 0 < x <= k + = x / k if x > k + ''' + def __init__(self, k_init=21.0): + super(SbPiPLU, self).__init__() + self.k = nn.Parameter(torch.tensor(k_init)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + softsign_x = F.softsign(x) + + part1 = 2 * softsign_x + softsign_x.pow(2) + part2 = x + part3 = x / self.k + + res = torch.where(x > self.k, part3, x) + res = torch.where(x <= 0, part1, res) + + return res + +class Model(nn.Module): + def __init__(self, k_init=21.0): + super(Model, self).__init__() + self.act = SbPiPLU(k_init=k_init) + + def forward(self, x): + return self.act(x) + +def get_inputs(): + input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 25.0 + return [input_tensor.contiguous()] + +def get_init_inputs(): + return [K_INIT] \ No newline at end of file diff --git a/S1/hli28146_#116/prompt.txt b/S1/hli28146_#116/prompt.txt new file mode 100644 index 00000000..2924e095 --- /dev/null +++ b/S1/hli28146_#116/prompt.txt @@ -0,0 +1,78 @@ +Write a custom CUDA kernel to optimize `Sb-PiPLU` with a trainable parameter `k`. + +Formula: + f(x) = 2*softsign(x) + softsign(x)^2 if x <= 0 + = x if 0 < x <= k + = x / k if x > k +where `k` is a learnable nn.Parameter. + +Problem Analysis: +1. Memory Bound & Computationally Heavy: The operation is element-wise but involves multiple branches and arithmetic operations. +2. Operator Chaining: PyTorch implementation requires multiple `torch.where` calls. +3. Trainable Parameter: The kernel must accept `k` as a scalar input that is determined at runtime from the `nn.Parameter`. + +Optimization Strategy: Fused Element-wise Kernel with Vectorization + +1. One-Thread-per-Element: Map each element to a CUDA thread. + +2. Vectorized Loads (float4): Use `float4` to process 128 bits per memory transaction. + +3. Fused Branching Logic: + - The scalar parameter `k` and `1/k` are passed to the kernel. + - Kernel logic uses a nested `if-else` to handle the three segments. + +4. One-Pass: Fuse all steps into a single read-compute-write kernel. + +Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is: + +```python +import torch +import torch.nn as nn +import torch.nn.functional as F + +BATCH_SIZE = 4096 +HIDDEN_DIM = 4096 +SHAPE = (BATCH_SIZE, HIDDEN_DIM) + +# Sb-PiPLU 初始参数 k +K_INIT = 21.0 + +class SbPiPLU(nn.Module): + ''' + Sb-PiPLU: A Novel Parametric Activation Function for Deep Learning + DOI:10.1109/ACCESS.2025.3561464 + Formula: + f(x) = 2*softsign(x) + softsign(x)^2 if x <= 0 + = x if 0 < x <= k + = x / k if x > k + ''' + def __init__(self, k_init=21.0): + super(SbPiPLU, self).__init__() + self.k = nn.Parameter(torch.tensor(k_init)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + softsign_x = F.softsign(x) + + part1 = 2 * softsign_x + softsign_x.pow(2) + part2 = x + part3 = x / self.k + + res = torch.where(x > self.k, part3, x) + res = torch.where(x <= 0, part1, res) + + return res + +class Model(nn.Module): + def __init__(self, k_init=21.0): + super(Model, self).__init__() + self.act = SbPiPLU(k_init=k_init) + + def forward(self, x): + return self.act(x) + +def get_inputs(): + input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 25.0 + return [input_tensor.contiguous()] + +def get_init_inputs(): + return [K_INIT] \ No newline at end of file diff --git a/S1/hli28146_#116/run_code.py b/S1/hli28146_#116/run_code.py new file mode 100644 index 00000000..25f1daae --- /dev/null +++ b/S1/hli28146_#116/run_code.py @@ -0,0 +1,74 @@ +########################################################### +# 性能和精度验证程序 +########################################################### +import torch +import torch.nn as nn +import time +from SbPiPLU_torch import Model,get_inputs,get_init_inputs +from SbPiPLU_cuda import ModelNew + +def run_benchmark(): + # 检查 CUDA 是否可用 + if not torch.cuda.is_available(): + print("CUDA 不可用，请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。") + return + else: + device = torch.device("cuda") + + # 初始化模型 + init_inputs = get_init_inputs() + init_inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs + ] + inputs = get_inputs() + inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs + ] + + torch_model = Model(*init_inputs).cuda() + cuda_model = ModelNew(*init_inputs).cuda() + + torch_model.eval() + cuda_model.eval() + + print("-------------------- 精度对齐验证 --------------------") + with torch.no_grad(): + output_torch = torch_model( *inputs) + output_cuda = cuda_model(*inputs) + + precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03) + if precision_flag: + print("✅ 精度对齐：两个模型的输出结果非常接近。") + else: + print("❌ 精度不一致！") + + print("\n-------------------- 性能加速比测试 --------------------") + num_iterations = 100 + + # PyTorch 模型计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = torch_model(*inputs) + torch.cuda.synchronize() + torch_time = (time.time() - start_time) / num_iterations + + # 自定义 CUDA 内核计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = cuda_model(*inputs) + torch.cuda.synchronize() + cuda_time = (time.time() - start_time) / num_iterations + + print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒") + print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒") + speedup = 0 + if cuda_time > 0: + speedup = torch_time / cuda_time + print(f"加速比 (Speedup): {speedup:.2f}x") + else: + print("CUDA 内核执行时间为0，无法计算加速比。") + return precision_flag,speedup +if __name__ == "__main__": + precision_flag,speedup = run_benchmark() \ No newline at end of file