forked from ccf-ai-infra/GPUCodeForces
finish ACON-C #25
This commit is contained in:
parent
f876a28ada
commit
70badfca3c
|
|
@ -0,0 +1,122 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.cpp_extension import load_inline
|
||||
|
||||
cpp_source = """
|
||||
#include <torch/extension.h>
|
||||
|
||||
torch::Tensor aconc_cuda_forward(const torch::Tensor& input, float p1, float p2, float beta);
|
||||
"""
|
||||
|
||||
cuda_source = """
|
||||
#include <torch/extension.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <math.h>
|
||||
|
||||
// Vectorized type for 128-bit access
|
||||
struct __align__(16) Float4 {
|
||||
float x, y, z, w;
|
||||
};
|
||||
|
||||
// Core computation logic (Device Inline)
|
||||
// Formula: (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x
|
||||
// Optimization: Precompute (p1 - p2) as 'pd' to save one mul per element.
|
||||
// f(x) = (pd * x) * sigmoid(beta * pd * x) + p2 * x
|
||||
__device__ __forceinline__ float aconc_op(float x, float p1, float p2, float beta, float pd) {
|
||||
float diff = pd * x; // (p1 - p2) * x
|
||||
|
||||
// sigmoid(val) = 1 / (1 + exp(-val))
|
||||
// Use __expf for fast approximation
|
||||
float arg = beta * diff;
|
||||
float sig = 1.0f / (1.0f + __expf(-arg));
|
||||
|
||||
return diff * sig + p2 * x;
|
||||
}
|
||||
|
||||
__global__ void aconc_kernel(
|
||||
const float* __restrict__ input,
|
||||
float* __restrict__ output,
|
||||
const float p1,
|
||||
const float p2,
|
||||
const float beta,
|
||||
const int n_elements)
|
||||
{
|
||||
int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int stride = blockDim.x * gridDim.x;
|
||||
|
||||
// Optimization: Compute the difference between p1 and p2 once
|
||||
// This reduces register pressure and instruction count inside the loop
|
||||
float pd = p1 - p2;
|
||||
|
||||
// 1. Vectorized Loop
|
||||
int vec_loops = n_elements / 4;
|
||||
const Float4* vec_input = reinterpret_cast<const Float4*>(input);
|
||||
Float4* vec_output = reinterpret_cast<Float4*>(output);
|
||||
|
||||
for (int i = idx; i < vec_loops; i += stride) {
|
||||
Float4 in_val = vec_input[i];
|
||||
Float4 out_val;
|
||||
|
||||
out_val.x = aconc_op(in_val.x, p1, p2, beta, pd);
|
||||
out_val.y = aconc_op(in_val.y, p1, p2, beta, pd);
|
||||
out_val.z = aconc_op(in_val.z, p1, p2, beta, pd);
|
||||
out_val.w = aconc_op(in_val.w, p1, p2, beta, pd);
|
||||
|
||||
vec_output[i] = out_val;
|
||||
}
|
||||
|
||||
// 2. Scalar Loop for tail
|
||||
int tail_start = vec_loops * 4;
|
||||
for (int i = tail_start + idx; i < n_elements; i += stride) {
|
||||
output[i] = aconc_op(input[i], p1, p2, beta, pd);
|
||||
}
|
||||
}
|
||||
|
||||
torch::Tensor aconc_cuda_forward(const torch::Tensor& input, float p1, float p2, float beta) {
|
||||
TORCH_CHECK(input.is_cuda(), "Input tensor must be a CUDA tensor");
|
||||
TORCH_CHECK(input.is_contiguous(), "Input tensor must be contiguous");
|
||||
|
||||
auto output = torch::empty_like(input);
|
||||
const int n_elements = input.numel();
|
||||
|
||||
const int block_size = 256;
|
||||
int grid_size = (n_elements + block_size * 4 - 1) / (block_size * 4);
|
||||
if (grid_size > 65535) grid_size = 65535;
|
||||
|
||||
aconc_kernel<<<grid_size, block_size>>>(
|
||||
input.data_ptr<float>(),
|
||||
output.data_ptr<float>(),
|
||||
p1, p2, beta,
|
||||
n_elements
|
||||
);
|
||||
|
||||
return output;
|
||||
}
|
||||
"""
|
||||
|
||||
aconc_op = load_inline(
|
||||
name='aconc_op',
|
||||
cpp_sources=cpp_source,
|
||||
cuda_sources=cuda_source,
|
||||
functions=['aconc_cuda_forward'],
|
||||
verbose=False,
|
||||
extra_cuda_cflags=['-O3']
|
||||
)
|
||||
|
||||
class AconCNew(nn.Module):
|
||||
def __init__(self, p1, p2, beta):
|
||||
super(AconCNew, self).__init__()
|
||||
self.p1 = p1
|
||||
self.p2 = p2
|
||||
self.beta = beta
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return aconc_op.aconc_cuda_forward(x, self.p1, self.p2, self.beta)
|
||||
|
||||
class ModelNew(nn.Module):
|
||||
def __init__(self, p1, p2, beta):
|
||||
super(ModelNew, self).__init__()
|
||||
self.act = AconCNew(p1, p2, beta)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.act(x)
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
BATCH_SIZE = 4096
|
||||
DIM = 4096
|
||||
SHAPE = (BATCH_SIZE, DIM)
|
||||
|
||||
# Default parameters for AconC
|
||||
P1_INIT = 1.0
|
||||
P2_INIT = 0.5
|
||||
BETA_INIT = 1.0
|
||||
|
||||
class AconC(nn.Module):
|
||||
"""
|
||||
AconC (Activate or Not) Activation Function.
|
||||
Formula: f(x) = (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x
|
||||
"""
|
||||
def __init__(self, p1=P1_INIT, p2=P2_INIT, beta=BETA_INIT):
|
||||
super(AconC, self).__init__()
|
||||
# In a real scenario, these might be learnable tensors.
|
||||
# For this kernel benchmark, we treat them as scalars to focus on operator fusion.
|
||||
self.p1 = p1
|
||||
self.p2 = p2
|
||||
self.beta = beta
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
diff = (self.p1 * x - self.p2 * x)
|
||||
return diff * torch.sigmoid(self.beta * diff) + self.p2 * x
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self, p1, p2, beta):
|
||||
super(Model, self).__init__()
|
||||
self.act = AconC(p1, p2, beta)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.act(x)
|
||||
|
||||
def get_inputs():
|
||||
x = torch.randn(SHAPE, dtype=torch.float32)
|
||||
return [x.contiguous()]
|
||||
|
||||
def get_init_inputs():
|
||||
return [P1_INIT, P2_INIT, BETA_INIT]
|
||||
|
|
@ -0,0 +1,71 @@
|
|||
Write a custom CUDA kernel to optimize the AconC (Activate or Not) activation function.
|
||||
|
||||
The mathematical definition is:
|
||||
f(x) = (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x
|
||||
where p1, p2, and beta are scalar parameters.
|
||||
|
||||
Problem Analysis:
|
||||
The standard PyTorch implementation is heavily memory-bound due to the complex arithmetic chain.
|
||||
1. It generates multiple intermediate tensors for terms like `p1*x`, `p2*x`, `p1*x - p2*x`, and the sigmoid result.
|
||||
2. It requires multiple passes over global memory to read inputs and write intermediate results, saturating memory bandwidth.
|
||||
3. The arithmetic intensity is relatively high for an activation function, involving exp, multiple multiplications, and additions.
|
||||
|
||||
Optimization Strategy: Fused Element-wise Kernel with Vectorized Access
|
||||
|
||||
1. Mathematical Simplification & Fusion: Simplify the expression in the kernel to reuse intermediate values stored in registers.
|
||||
Let diff = (p1 - p2) * x
|
||||
Result = diff * sigmoid(beta * diff) + p2 * x
|
||||
This avoids re-reading x or re-computing the difference.
|
||||
|
||||
2. Vectorized Memory Access: Use float4 data types to load and store 128 bits (4 floats) per instruction. This is crucial for hiding the latency of the arithmetic operations (especially exp).
|
||||
|
||||
3. Fast Math Intrinsics: Use `__expf` for the sigmoid calculation `1.0 / (1.0 + __expf(-val))` to speed up the transcendental part.
|
||||
|
||||
4. Grid-Stride Loop: Implement a robust grid-stride loop to handle any tensor size efficiently.
|
||||
|
||||
Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is:
|
||||
|
||||
```python
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
BATCH_SIZE = 4096
|
||||
DIM = 4096
|
||||
SHAPE = (BATCH_SIZE, DIM)
|
||||
|
||||
# Default parameters for AconC
|
||||
P1_INIT = 1.0
|
||||
P2_INIT = 0.5
|
||||
BETA_INIT = 1.0
|
||||
|
||||
class AconC(nn.Module):
|
||||
"""
|
||||
AconC (Activate or Not) Activation Function.
|
||||
Formula: f(x) = (p1*x - p2*x) * sigmoid(beta * (p1*x - p2*x)) + p2*x
|
||||
"""
|
||||
def __init__(self, p1=P1_INIT, p2=P2_INIT, beta=BETA_INIT):
|
||||
super(AconC, self).__init__()
|
||||
# In a real scenario, these might be learnable tensors.
|
||||
# For this kernel benchmark, we treat them as scalars to focus on operator fusion.
|
||||
self.p1 = p1
|
||||
self.p2 = p2
|
||||
self.beta = beta
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
diff = (self.p1 * x - self.p2 * x)
|
||||
return diff * torch.sigmoid(self.beta * diff) + self.p2 * x
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self, p1, p2, beta):
|
||||
super(Model, self).__init__()
|
||||
self.act = AconC(p1, p2, beta)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return self.act(x)
|
||||
|
||||
def get_inputs():
|
||||
x = torch.randn(SHAPE, dtype=torch.float32)
|
||||
return [x.contiguous()]
|
||||
|
||||
def get_init_inputs():
|
||||
return [P1_INIT, P2_INIT, BETA_INIT]
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
###########################################################
|
||||
# 性能和精度验证程序
|
||||
###########################################################
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import time
|
||||
from aconc_torch import Model,get_inputs,get_init_inputs
|
||||
from aconc_cuda import ModelNew
|
||||
|
||||
def run_benchmark():
|
||||
# 检查 CUDA 是否可用
|
||||
if not torch.cuda.is_available():
|
||||
print("CUDA 不可用，请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。")
|
||||
return
|
||||
else:
|
||||
device = torch.device("cuda")
|
||||
|
||||
# 初始化模型
|
||||
init_inputs = get_init_inputs()
|
||||
init_inputs = [
|
||||
x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs
|
||||
]
|
||||
inputs = get_inputs()
|
||||
inputs = [
|
||||
x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs
|
||||
]
|
||||
|
||||
torch_model = Model(*init_inputs).cuda()
|
||||
cuda_model = ModelNew(*init_inputs).cuda()
|
||||
|
||||
torch_model.eval()
|
||||
cuda_model.eval()
|
||||
|
||||
print("-------------------- 精度对齐验证 --------------------")
|
||||
with torch.no_grad():
|
||||
output_torch = torch_model( *inputs)
|
||||
output_cuda = cuda_model(*inputs)
|
||||
|
||||
precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03)
|
||||
if precision_flag:
|
||||
print("✅ 精度对齐：两个模型的输出结果非常接近。")
|
||||
else:
|
||||
print("❌ 精度不一致！")
|
||||
|
||||
print("\n-------------------- 性能加速比测试 --------------------")
|
||||
num_iterations = 100
|
||||
|
||||
# PyTorch 模型计时
|
||||
torch.cuda.synchronize()
|
||||
start_time = time.time()
|
||||
for _ in range(num_iterations):
|
||||
_ = torch_model(*inputs)
|
||||
torch.cuda.synchronize()
|
||||
torch_time = (time.time() - start_time) / num_iterations
|
||||
|
||||
# 自定义 CUDA 内核计时
|
||||
torch.cuda.synchronize()
|
||||
start_time = time.time()
|
||||
for _ in range(num_iterations):
|
||||
_ = cuda_model(*inputs)
|
||||
torch.cuda.synchronize()
|
||||
cuda_time = (time.time() - start_time) / num_iterations
|
||||
|
||||
print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒")
|
||||
print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒")
|
||||
speedup = 0
|
||||
if cuda_time > 0:
|
||||
speedup = torch_time / cuda_time
|
||||
print(f"加速比 (Speedup): {speedup:.2f}x")
|
||||
else:
|
||||
print("CUDA 内核执行时间为0，无法计算加速比。")
|
||||
return precision_flag,speedup
|
||||
if __name__ == "__main__":
|
||||
precision_flag,speedup = run_benchmark()
|
||||
Loading…
Reference in New Issue