From 70badfca3c5016d3e3eaa6ba0ac065e8bc235cbb Mon Sep 17 00:00:00 2001 From: hli28146 Date: Tue, 2 Dec 2025 15:30:53 +0800 Subject: [PATCH] finish ACON-C #25 --- S1/hli28146_#25/aconc_cuda.py | 122 +++++++++++++++++++++++++++++++++ S1/hli28146_#25/aconc_torch.py | 43 ++++++++++++ S1/hli28146_#25/prompt.txt | 71 +++++++++++++++++++ S1/hli28146_#25/run_code.py | 74 ++++++++++++++++++++ 4 files changed, 310 insertions(+) create mode 100644 S1/hli28146_#25/aconc_cuda.py create mode 100644 S1/hli28146_#25/aconc_torch.py create mode 100644 S1/hli28146_#25/prompt.txt create mode 100644 S1/hli28146_#25/run_code.py diff --git a/S1/hli28146_#25/aconc_cuda.py b/S1/hli28146_#25/aconc_cuda.py new file mode 100644 index 00000000..89b580c4 --- /dev/null +++ b/S1/hli28146_#25/aconc_cuda.py @@ -0,0 +1,122 @@ +import torch +import torch.nn as nn +from torch.utils.cpp_extension import load_inline + +cpp_source = """ +#include + +torch::Tensor aconc_cuda_forward(const torch::Tensor& input, float p1, float p2, float beta); +""" + +cuda_source = """ +#include +#include +#include + +// Vectorized type for 128-bit access +struct __align__(16) Float4 { + float x, y, z, w; +}; + +// Core computation logic (Device Inline) +// Formula: (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x +// Optimization: Precompute (p1 - p2) as 'pd' to save one mul per element. +// f(x) = (pd * x) * sigmoid(beta * pd * x) + p2 * x +__device__ __forceinline__ float aconc_op(float x, float p1, float p2, float beta, float pd) { + float diff = pd * x; // (p1 - p2) * x + + // sigmoid(val) = 1 / (1 + exp(-val)) + // Use __expf for fast approximation + float arg = beta * diff; + float sig = 1.0f / (1.0f + __expf(-arg)); + + return diff * sig + p2 * x; +} + +__global__ void aconc_kernel( + const float* __restrict__ input, + float* __restrict__ output, + const float p1, + const float p2, + const float beta, + const int n_elements) +{ + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int stride = blockDim.x * gridDim.x; + + // Optimization: Compute the difference between p1 and p2 once + // This reduces register pressure and instruction count inside the loop + float pd = p1 - p2; + + // 1. Vectorized Loop + int vec_loops = n_elements / 4; + const Float4* vec_input = reinterpret_cast(input); + Float4* vec_output = reinterpret_cast(output); + + for (int i = idx; i < vec_loops; i += stride) { + Float4 in_val = vec_input[i]; + Float4 out_val; + + out_val.x = aconc_op(in_val.x, p1, p2, beta, pd); + out_val.y = aconc_op(in_val.y, p1, p2, beta, pd); + out_val.z = aconc_op(in_val.z, p1, p2, beta, pd); + out_val.w = aconc_op(in_val.w, p1, p2, beta, pd); + + vec_output[i] = out_val; + } + + // 2. Scalar Loop for tail + int tail_start = vec_loops * 4; + for (int i = tail_start + idx; i < n_elements; i += stride) { + output[i] = aconc_op(input[i], p1, p2, beta, pd); + } +} + +torch::Tensor aconc_cuda_forward(const torch::Tensor& input, float p1, float p2, float beta) { + TORCH_CHECK(input.is_cuda(), "Input tensor must be a CUDA tensor"); + TORCH_CHECK(input.is_contiguous(), "Input tensor must be contiguous"); + + auto output = torch::empty_like(input); + const int n_elements = input.numel(); + + const int block_size = 256; + int grid_size = (n_elements + block_size * 4 - 1) / (block_size * 4); + if (grid_size > 65535) grid_size = 65535; + + aconc_kernel<<>>( + input.data_ptr(), + output.data_ptr(), + p1, p2, beta, + n_elements + ); + + return output; +} +""" + +aconc_op = load_inline( + name='aconc_op', + cpp_sources=cpp_source, + cuda_sources=cuda_source, + functions=['aconc_cuda_forward'], + verbose=False, + extra_cuda_cflags=['-O3'] +) + +class AconCNew(nn.Module): + def __init__(self, p1, p2, beta): + super(AconCNew, self).__init__() + self.p1 = p1 + self.p2 = p2 + self.beta = beta + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return aconc_op.aconc_cuda_forward(x, self.p1, self.p2, self.beta) + +class ModelNew(nn.Module): + def __init__(self, p1, p2, beta): + super(ModelNew, self).__init__() + self.act = AconCNew(p1, p2, beta) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.act(x) \ No newline at end of file diff --git a/S1/hli28146_#25/aconc_torch.py b/S1/hli28146_#25/aconc_torch.py new file mode 100644 index 00000000..d0f6f75d --- /dev/null +++ b/S1/hli28146_#25/aconc_torch.py @@ -0,0 +1,43 @@ +import torch +import torch.nn as nn + +BATCH_SIZE = 4096 +DIM = 4096 +SHAPE = (BATCH_SIZE, DIM) + +# Default parameters for AconC +P1_INIT = 1.0 +P2_INIT = 0.5 +BETA_INIT = 1.0 + +class AconC(nn.Module): + """ + AconC (Activate or Not) Activation Function. + Formula: f(x) = (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x + """ + def __init__(self, p1=P1_INIT, p2=P2_INIT, beta=BETA_INIT): + super(AconC, self).__init__() + # In a real scenario, these might be learnable tensors. + # For this kernel benchmark, we treat them as scalars to focus on operator fusion. + self.p1 = p1 + self.p2 = p2 + self.beta = beta + + def forward(self, x: torch.Tensor) -> torch.Tensor: + diff = (self.p1 * x - self.p2 * x) + return diff * torch.sigmoid(self.beta * diff) + self.p2 * x + +class Model(nn.Module): + def __init__(self, p1, p2, beta): + super(Model, self).__init__() + self.act = AconC(p1, p2, beta) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.act(x) + +def get_inputs(): + x = torch.randn(SHAPE, dtype=torch.float32) + return [x.contiguous()] + +def get_init_inputs(): + return [P1_INIT, P2_INIT, BETA_INIT] \ No newline at end of file diff --git a/S1/hli28146_#25/prompt.txt b/S1/hli28146_#25/prompt.txt new file mode 100644 index 00000000..f71f8444 --- /dev/null +++ b/S1/hli28146_#25/prompt.txt @@ -0,0 +1,71 @@ +Write a custom CUDA kernel to optimize the AconC (Activate or Not) activation function. + +The mathematical definition is: +f(x) = (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x +where p1, p2, and beta are scalar parameters. + +Problem Analysis: +The standard PyTorch implementation is heavily memory-bound due to the complex arithmetic chain. +1. It generates multiple intermediate tensors for terms like `p1*x`, `p2*x`, `p1*x - p2*x`, and the sigmoid result. +2. It requires multiple passes over global memory to read inputs and write intermediate results, saturating memory bandwidth. +3. The arithmetic intensity is relatively high for an activation function, involving exp, multiple multiplications, and additions. + +Optimization Strategy: Fused Element-wise Kernel with Vectorized Access + +1. Mathematical Simplification & Fusion: Simplify the expression in the kernel to reuse intermediate values stored in registers. + Let diff = (p1 - p2) * x + Result = diff * sigmoid(beta * diff) + p2 * x + This avoids re-reading x or re-computing the difference. + +2. Vectorized Memory Access: Use float4 data types to load and store 128 bits (4 floats) per instruction. This is crucial for hiding the latency of the arithmetic operations (especially exp). + +3. Fast Math Intrinsics: Use `__expf` for the sigmoid calculation `1.0 / (1.0 + __expf(-val))` to speed up the transcendental part. + +4. Grid-Stride Loop: Implement a robust grid-stride loop to handle any tensor size efficiently. + +Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is: + +```python +import torch +import torch.nn as nn + +BATCH_SIZE = 4096 +DIM = 4096 +SHAPE = (BATCH_SIZE, DIM) + +# Default parameters for AconC +P1_INIT = 1.0 +P2_INIT = 0.5 +BETA_INIT = 1.0 + +class AconC(nn.Module): + """ + AconC (Activate or Not) Activation Function. + Formula: f(x) = (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x + """ + def __init__(self, p1=P1_INIT, p2=P2_INIT, beta=BETA_INIT): + super(AconC, self).__init__() + # In a real scenario, these might be learnable tensors. + # For this kernel benchmark, we treat them as scalars to focus on operator fusion. + self.p1 = p1 + self.p2 = p2 + self.beta = beta + + def forward(self, x: torch.Tensor) -> torch.Tensor: + diff = (self.p1 * x - self.p2 * x) + return diff * torch.sigmoid(self.beta * diff) + self.p2 * x + +class Model(nn.Module): + def __init__(self, p1, p2, beta): + super(Model, self).__init__() + self.act = AconC(p1, p2, beta) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return self.act(x) + +def get_inputs(): + x = torch.randn(SHAPE, dtype=torch.float32) + return [x.contiguous()] + +def get_init_inputs(): + return [P1_INIT, P2_INIT, BETA_INIT] \ No newline at end of file diff --git a/S1/hli28146_#25/run_code.py b/S1/hli28146_#25/run_code.py new file mode 100644 index 00000000..71d7c8ca --- /dev/null +++ b/S1/hli28146_#25/run_code.py @@ -0,0 +1,74 @@ +########################################################### +# 性能和精度验证程序 +########################################################### +import torch +import torch.nn as nn +import time +from aconc_torch import Model,get_inputs,get_init_inputs +from aconc_cuda import ModelNew + +def run_benchmark(): + # 检查 CUDA 是否可用 + if not torch.cuda.is_available(): + print("CUDA 不可用，请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。") + return + else: + device = torch.device("cuda") + + # 初始化模型 + init_inputs = get_init_inputs() + init_inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs + ] + inputs = get_inputs() + inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs + ] + + torch_model = Model(*init_inputs).cuda() + cuda_model = ModelNew(*init_inputs).cuda() + + torch_model.eval() + cuda_model.eval() + + print("-------------------- 精度对齐验证 --------------------") + with torch.no_grad(): + output_torch = torch_model( *inputs) + output_cuda = cuda_model(*inputs) + + precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03) + if precision_flag: + print("✅ 精度对齐：两个模型的输出结果非常接近。") + else: + print("❌ 精度不一致！") + + print("\n-------------------- 性能加速比测试 --------------------") + num_iterations = 100 + + # PyTorch 模型计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = torch_model(*inputs) + torch.cuda.synchronize() + torch_time = (time.time() - start_time) / num_iterations + + # 自定义 CUDA 内核计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = cuda_model(*inputs) + torch.cuda.synchronize() + cuda_time = (time.time() - start_time) / num_iterations + + print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒") + print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒") + speedup = 0 + if cuda_time > 0: + speedup = torch_time / cuda_time + print(f"加速比 (Speedup): {speedup:.2f}x") + else: + print("CUDA 内核执行时间为0，无法计算加速比。") + return precision_flag,speedup +if __name__ == "__main__": + precision_flag,speedup = run_benchmark() \ No newline at end of file