finish Sb-PiPLU #116

S1/hli28146_#116/SbPiPLU_cuda.py

@@ -0,0 +1,115 @@
+import torch
+import torch.nn as nn
+from torch.utils.cpp_extension import load_inline
+
+cpp_source = """
+#include <torch/extension.h>
+
+torch::Tensor sb_piplu_cuda_forward(const torch::Tensor& input, const torch::Tensor& k_tensor);
+"""
+
+cuda_source = """
+#include <torch/extension.h>
+#include <cuda_runtime.h>
+#include <math.h>
+
+#define BLOCK_SIZE 256
+
+struct __align__(16) Float4 {
+    float x, y, z, w;
+};
+
+// Sb-PiPLU Logic
+__device__ __forceinline__ float compute_sb_piplu(float x, float k, float inv_k) {
+    if (x > k) {
+        return x * inv_k;
+    } else if (x > 0.0f) { // 0 < x <= k
+        return x;
+    } else { // x <= 0
+        float ss = x / (1.0f + fabsf(x));
+        return 2.0f * ss + ss * ss;
+    }
+}
+
+__global__ void sb_piplu_kernel(
+    float* __restrict__ output,
+    const float* __restrict__ input,
+    const int n,
+    const float k,
+    const float inv_k) 
+{
+    const int idx = blockIdx.x * blockDim.x + threadIdx.x;
+    const int vec_n = n / 4;
+    
+    int i = idx;
+    const int stride = blockDim.x * gridDim.x;
+
+    for (; i < vec_n; i += stride) {
+        Float4 in_vec = reinterpret_cast<const Float4*>(input)[i];
+        Float4 out_vec;
+
+        out_vec.x = compute_sb_piplu(in_vec.x, k, inv_k);
+        out_vec.y = compute_sb_piplu(in_vec.y, k, inv_k);
+        out_vec.z = compute_sb_piplu(in_vec.z, k, inv_k);
+        out_vec.w = compute_sb_piplu(in_vec.w, k, inv_k);
+
+        reinterpret_cast<Float4*>(output)[i] = out_vec;
+    }
+
+    int start_scalar = vec_n * 4;
+    int global_tid = blockIdx.x * blockDim.x + threadIdx.x;
+    int total_threads = gridDim.x * gridDim.x;
+    
+    int current_idx = start_scalar + global_tid;
+    while (current_idx < n) {
+        output[current_idx] = compute_sb_piplu(input[current_idx], k, inv_k);
+        current_idx += total_threads;
+    }
+}
+
+torch::Tensor sb_piplu_cuda_forward(const torch::Tensor& input, const torch::Tensor& k_tensor) {
+    TORCH_CHECK(input.is_cuda(), "Input must be a CUDA tensor");
+    TORCH_CHECK(input.is_contiguous(), "Input must be contiguous");
+    
+    const int n = input.numel();
+    auto output = torch::empty_like(input);
+    
+    // Extract scalar value from k_tensor on the host
+    const float k = k_tensor.item<float>();
+    const float inv_k = 1.0f / k;
+
+    const int vec_n = n / 4;
+    const int grid_size = (vec_n + BLOCK_SIZE - 1) / BLOCK_SIZE;
+    
+    int final_grid = (grid_size < 1) ? 1 : grid_size;
+    if (final_grid > 65535) final_grid = 65535;
+
+    sb_piplu_kernel<<<final_grid, BLOCK_SIZE>>>(
+        output.data_ptr<float>(),
+        input.data_ptr<float>(),
+        n,
+        k,
+        inv_k
+    );
+
+    return output;
+}
+"""
+
+sb_piplu_op_module = load_inline(
+    name='sb_piplu_param_op',
+    cpp_sources=cpp_source,
+    cuda_sources=cuda_source,
+    functions=['sb_piplu_cuda_forward'],
+    verbose=False,
+    extra_cuda_cflags=['-O3']
+)
+
+class ModelNew(nn.Module):
+    def __init__(self, k_init=21.0):
+        super(ModelNew, self).__init__()
+        self.k = nn.Parameter(torch.tensor(k_init))
+        self.op = sb_piplu_op_module
+
+    def forward(self, input_tensor: torch.Tensor) -> torch.Tensor:
+        return self.op.sb_piplu_cuda_forward(input_tensor.contiguous(), self.k)

S1/hli28146_#116/SbPiPLU_torch.py

@@ -0,0 +1,50 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+BATCH_SIZE = 4096
+HIDDEN_DIM = 4096
+SHAPE = (BATCH_SIZE, HIDDEN_DIM)
+
+# Sb-PiPLU 初始参数 k
+K_INIT = 21.0
+
+class SbPiPLU(nn.Module):
+    '''
+    Sb-PiPLU: A Novel Parametric Activation Function for Deep Learning
+    DOI:10.1109/ACCESS.2025.3561464
+    Formula:
+      f(x) = 2*softsign(x) + softsign(x)^2    if x <= 0
+           = x                                if 0 < x <= k
+           = x / k                            if x > k
+    '''
+    def __init__(self, k_init=21.0):
+        super(SbPiPLU, self).__init__()
+        self.k = nn.Parameter(torch.tensor(k_init))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        softsign_x = F.softsign(x)
+        
+        part1 = 2 * softsign_x + softsign_x.pow(2)
+        part2 = x
+        part3 = x / self.k
+        
+        res = torch.where(x > self.k, part3, x)
+        res = torch.where(x <= 0, part1, res)
+        
+        return res
+
+class Model(nn.Module):
+    def __init__(self, k_init=21.0):
+        super(Model, self).__init__()
+        self.act = SbPiPLU(k_init=k_init)
+    
+    def forward(self, x):
+        return self.act(x)
+
+def get_inputs():
+    input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 25.0
+    return [input_tensor.contiguous()]
+
+def get_init_inputs():
+    return [K_INIT]

S1/hli28146_#116/prompt.txt

@@ -0,0 +1,78 @@
+Write a custom CUDA kernel to optimize `Sb-PiPLU` with a trainable parameter `k`.
+
+Formula:
+  f(x) = 2*softsign(x) + softsign(x)^2   if x <= 0
+       = x                                if 0 < x <= k
+       = x / k                            if x > k
+where `k` is a learnable nn.Parameter.
+
+Problem Analysis:
+1. Memory Bound & Computationally Heavy: The operation is element-wise but involves multiple branches and arithmetic operations.
+2. Operator Chaining: PyTorch implementation requires multiple `torch.where` calls.
+3. Trainable Parameter: The kernel must accept `k` as a scalar input that is determined at runtime from the `nn.Parameter`.
+
+Optimization Strategy: Fused Element-wise Kernel with Vectorization
+
+1. One-Thread-per-Element: Map each element to a CUDA thread.
+
+2. Vectorized Loads (float4): Use `float4` to process 128 bits per memory transaction.
+
+3. Fused Branching Logic:
+   - The scalar parameter `k` and `1/k` are passed to the kernel.
+   - Kernel logic uses a nested `if-else` to handle the three segments.
+
+4. One-Pass: Fuse all steps into a single read-compute-write kernel.
+  
+Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is:   
+  
+```python
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+BATCH_SIZE = 4096
+HIDDEN_DIM = 4096
+SHAPE = (BATCH_SIZE, HIDDEN_DIM)
+
+# Sb-PiPLU 初始参数 k
+K_INIT = 21.0
+
+class SbPiPLU(nn.Module):
+    '''
+    Sb-PiPLU: A Novel Parametric Activation Function for Deep Learning
+    DOI:10.1109/ACCESS.2025.3561464
+    Formula:
+      f(x) = 2*softsign(x) + softsign(x)^2    if x <= 0
+           = x                                if 0 < x <= k
+           = x / k                            if x > k
+    '''
+    def __init__(self, k_init=21.0):
+        super(SbPiPLU, self).__init__()
+        self.k = nn.Parameter(torch.tensor(k_init))
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        softsign_x = F.softsign(x)
+        
+        part1 = 2 * softsign_x + softsign_x.pow(2)
+        part2 = x
+        part3 = x / self.k
+        
+        res = torch.where(x > self.k, part3, x)
+        res = torch.where(x <= 0, part1, res)
+        
+        return res
+
+class Model(nn.Module):
+    def __init__(self, k_init=21.0):
+        super(Model, self).__init__()
+        self.act = SbPiPLU(k_init=k_init)
+    
+    def forward(self, x):
+        return self.act(x)
+
+def get_inputs():
+    input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 25.0
+    return [input_tensor.contiguous()]
+
+def get_init_inputs():
+    return [K_INIT]

S1/hli28146_#116/run_code.py

@@ -0,0 +1,74 @@
+###########################################################
+# 性能和精度验证程序
+###########################################################
+import torch
+import torch.nn as nn
+import time
+from SbPiPLU_torch import Model,get_inputs,get_init_inputs
+from SbPiPLU_cuda import ModelNew
+
+def run_benchmark():
+    # 检查 CUDA 是否可用
+    if not torch.cuda.is_available():
+        print("CUDA 不可用，请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。")
+        return
+    else:
+        device = torch.device("cuda")
+
+    # 初始化模型
+    init_inputs = get_init_inputs()
+    init_inputs = [
+        x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs
+    ]
+    inputs = get_inputs()
+    inputs = [
+        x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs
+    ]
+
+    torch_model = Model(*init_inputs).cuda()
+    cuda_model = ModelNew(*init_inputs).cuda()
+
+    torch_model.eval()
+    cuda_model.eval()
+
+    print("-------------------- 精度对齐验证 --------------------")
+    with torch.no_grad():
+        output_torch = torch_model( *inputs)
+        output_cuda = cuda_model(*inputs)
+
+    precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03)
+    if precision_flag:
+        print("✅ 精度对齐：两个模型的输出结果非常接近。")
+    else:
+        print("❌ 精度不一致！")
+        
+    print("\n-------------------- 性能加速比测试 --------------------")
+    num_iterations = 100 
+    
+    # PyTorch 模型计时
+    torch.cuda.synchronize()
+    start_time = time.time()
+    for _ in range(num_iterations):
+        _ = torch_model(*inputs)
+    torch.cuda.synchronize()
+    torch_time = (time.time() - start_time) / num_iterations
+    
+    # 自定义 CUDA 内核计时
+    torch.cuda.synchronize()
+    start_time = time.time()
+    for _ in range(num_iterations):
+        _ = cuda_model(*inputs)
+    torch.cuda.synchronize()
+    cuda_time = (time.time() - start_time) / num_iterations
+    
+    print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒")
+    print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒")
+    speedup = 0
+    if cuda_time > 0:
+        speedup = torch_time / cuda_time
+        print(f"加速比 (Speedup): {speedup:.2f}x")
+    else:
+        print("CUDA 内核执行时间为0，无法计算加速比。")
+    return precision_flag,speedup
+if __name__ == "__main__":
+    precision_flag,speedup = run_benchmark()