forked from ccf-ai-infra/GPUCodeForces
finish MMReLU #94
This commit is contained in:
parent
f876a28ada
commit
b4d8b2a5af
|
|
@ -0,0 +1,120 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
from torch.utils.cpp_extension import load_inline
|
||||
import math
|
||||
|
||||
cpp_source = """
|
||||
#include <torch/extension.h>
|
||||
|
||||
torch::Tensor mmrelu_cuda_forward(const torch::Tensor& input, float t, float sqrt_3t, float coeff);
|
||||
"""
|
||||
|
||||
cuda_source = """
|
||||
#include <torch/extension.h>
|
||||
#include <cuda_runtime.h>
|
||||
#include <math.h>
|
||||
|
||||
#define BLOCK_SIZE 256
|
||||
|
||||
struct __align__(16) Float4 {
|
||||
float x, y, z, w;
|
||||
};
|
||||
|
||||
// MMReLU Logic
|
||||
__device__ __forceinline__ float compute_mmrelu(float x, float t, float neg_sqrt_3t, float coeff) {
|
||||
if (x >= 0.0f) {
|
||||
return x;
|
||||
} else if (x > neg_sqrt_3t) { // -sqrt(3t) < x < 0
|
||||
return coeff * x * x + x;
|
||||
} else { // x <= -sqrt(3t)
|
||||
return t / x;
|
||||
}
|
||||
}
|
||||
|
||||
__global__ void mmrelu_kernel(
|
||||
float* __restrict__ output,
|
||||
const float* __restrict__ input,
|
||||
const int n,
|
||||
const float t,
|
||||
const float neg_sqrt_3t,
|
||||
const float coeff)
|
||||
{
|
||||
const int idx = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
const int vec_n = n / 4;
|
||||
|
||||
int i = idx;
|
||||
const int stride = blockDim.x * gridDim.x;
|
||||
|
||||
for (; i < vec_n; i += stride) {
|
||||
Float4 in_vec = reinterpret_cast<const Float4*>(input)[i];
|
||||
Float4 out_vec;
|
||||
|
||||
out_vec.x = compute_mmrelu(in_vec.x, t, neg_sqrt_3t, coeff);
|
||||
out_vec.y = compute_mmrelu(in_vec.y, t, neg_sqrt_3t, coeff);
|
||||
out_vec.z = compute_mmrelu(in_vec.z, t, neg_sqrt_3t, coeff);
|
||||
out_vec.w = compute_mmrelu(in_vec.w, t, neg_sqrt_3t, coeff);
|
||||
|
||||
reinterpret_cast<Float4*>(output)[i] = out_vec;
|
||||
}
|
||||
|
||||
int start_scalar = vec_n * 4;
|
||||
int global_tid = blockIdx.x * blockDim.x + threadIdx.x;
|
||||
int total_threads = gridDim.x * gridDim.x;
|
||||
|
||||
int current_idx = start_scalar + global_tid;
|
||||
while (current_idx < n) {
|
||||
output[current_idx] = compute_mmrelu(input[current_idx], t, neg_sqrt_3t, coeff);
|
||||
current_idx += total_threads;
|
||||
}
|
||||
}
|
||||
|
||||
torch::Tensor mmrelu_cuda_forward(const torch::Tensor& input, float t, float sqrt_3t, float coeff) {
|
||||
TORCH_CHECK(input.is_cuda(), "Input must be a CUDA tensor");
|
||||
TORCH_CHECK(input.is_contiguous(), "Input must be contiguous");
|
||||
|
||||
const int n = input.numel();
|
||||
auto output = torch::empty_like(input);
|
||||
|
||||
const int vec_n = n / 4;
|
||||
const int grid_size = (vec_n + BLOCK_SIZE - 1) / BLOCK_SIZE;
|
||||
|
||||
int final_grid = (grid_size < 1) ? 1 : grid_size;
|
||||
if (final_grid > 65535) final_grid = 65535;
|
||||
|
||||
float neg_sqrt_3t = -sqrt_3t;
|
||||
|
||||
mmrelu_kernel<<<final_grid, BLOCK_SIZE>>>(
|
||||
output.data_ptr<float>(),
|
||||
input.data_ptr<float>(),
|
||||
n,
|
||||
t,
|
||||
neg_sqrt_3t,
|
||||
coeff
|
||||
);
|
||||
|
||||
return output;
|
||||
}
|
||||
"""
|
||||
|
||||
mmrelu_op_module = load_inline(
|
||||
name='mmrelu_op',
|
||||
cpp_sources=cpp_source,
|
||||
cuda_sources=cuda_source,
|
||||
functions=['mmrelu_cuda_forward'],
|
||||
verbose=False,
|
||||
extra_cuda_cflags=['-O3']
|
||||
)
|
||||
|
||||
class ModelNew(nn.Module):
|
||||
def __init__(self, t=1.0):
|
||||
super(ModelNew, self).__init__()
|
||||
self.t = t
|
||||
# Pre-compute constants
|
||||
self.sqrt_3t = math.sqrt(3 * t)
|
||||
self.coeff = (2 * self.sqrt_3t) / (9 * t)
|
||||
self.op = mmrelu_op_module
|
||||
|
||||
def forward(self, input_tensor: torch.Tensor) -> torch.Tensor:
|
||||
return self.op.mmrelu_cuda_forward(
|
||||
input_tensor.contiguous(), self.t, self.sqrt_3t, self.coeff
|
||||
)
|
||||
|
|
@ -0,0 +1,55 @@
|
|||
import torch
|
||||
import torch.nn as nn
|
||||
import math
|
||||
|
||||
BATCH_SIZE = 4096
|
||||
HIDDEN_DIM = 4096
|
||||
SHAPE = (BATCH_SIZE, HIDDEN_DIM)
|
||||
|
||||
T_VALUE = 1.0
|
||||
|
||||
class MMReLU(nn.Module):
|
||||
'''
|
||||
MMReLU: A Simple and Smooth Activation Function with High Convergence Speed
|
||||
https://ieeexplore.ieee.org/document/9674529
|
||||
Formula:
|
||||
f(x) = x if x >= 0
|
||||
= (2*sqrt(3t)/(9t))*x^2 + x if -sqrt(3t) < x < 0
|
||||
= t / x if x <= -sqrt(3t)
|
||||
'''
|
||||
def __init__(self, t=1.0):
|
||||
super(MMReLU, self).__init__()
|
||||
self.t = t
|
||||
self.sqrt_3t = math.sqrt(3 * t)
|
||||
self.coeff = (2 * self.sqrt_3t) / (9 * t) if t > 0 else 0.0
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# Part 1: x >= 0 -> x
|
||||
# Part 2: -sqrt(3t) < x < 0 -> quadratic
|
||||
# Part 3: x <= -sqrt(3t) -> reciprocal
|
||||
|
||||
part1 = x
|
||||
part2 = self.coeff * x.pow(2) + x
|
||||
part3 = self.t / (x + 1e-9) # Add epsilon for stability
|
||||
|
||||
neg_res = torch.where(x > -self.sqrt_3t, part2, part3)
|
||||
|
||||
res = torch.where(x >= 0, part1, neg_res)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self, t=1.0):
|
||||
super(Model, self).__init__()
|
||||
self.act = MMReLU(t=t)
|
||||
|
||||
def forward(self, x):
|
||||
return self.act(x)
|
||||
|
||||
def get_inputs():
|
||||
input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 2.0
|
||||
return [input_tensor.contiguous()]
|
||||
|
||||
def get_init_inputs():
|
||||
return [T_VALUE]
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
Write a custom CUDA kernel to optimize `MMReLU`.
|
||||
|
||||
Formula:
|
||||
f(x) = x if x >= 0
|
||||
= (2*sqrt(3t)/(9t))*x^2 + x if -sqrt(3t) < x < 0
|
||||
= t / x if x <= -sqrt(3t)
|
||||
|
||||
Problem Analysis:
|
||||
1. Memory Bound: This is an element-wise activation with multiple branches.
|
||||
2. Operator Chaining: PyTorch implementation requires multiple `torch.where` calls and arithmetic operations, creating high memory traffic.
|
||||
|
||||
Optimization Strategy: Fused Element-wise Kernel with Vectorization
|
||||
|
||||
1. One-Thread-per-Element: Map each element to a CUDA thread.
|
||||
|
||||
2. Vectorized Loads (float4): Use `float4` to process 128 bits per memory transaction.
|
||||
|
||||
3. Fused Branching Logic:
|
||||
- Pre-compute constants like `sqrt(3t)` and coefficients on the host.
|
||||
- Kernel logic: Use a nested `if-else` to handle the three segments.
|
||||
- All computations are fused in registers.
|
||||
|
||||
4. One-Pass: Fuse all steps into a single read-compute-write kernel.
|
||||
|
||||
Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is:
|
||||
|
||||
```python
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import math
|
||||
|
||||
BATCH_SIZE = 4096
|
||||
HIDDEN_DIM = 4096
|
||||
SHAPE = (BATCH_SIZE, HIDDEN_DIM)
|
||||
|
||||
T_VALUE = 1.0
|
||||
|
||||
class MMReLU(nn.Module):
|
||||
'''
|
||||
MMReLU: A Simple and Smooth Activation Function with High Convergence Speed
|
||||
https://ieeexplore.ieee.org/document/9674529
|
||||
Formula:
|
||||
f(x) = x if x >= 0
|
||||
= (2*sqrt(3t)/(9t))*x^2 + x if -sqrt(3t) < x < 0
|
||||
= t / x if x <= -sqrt(3t)
|
||||
'''
|
||||
def __init__(self, t=1.0):
|
||||
super(MMReLU, self).__init__()
|
||||
self.t = t
|
||||
self.sqrt_3t = math.sqrt(3 * t)
|
||||
self.coeff = (2 * self.sqrt_3t) / (9 * t) if t > 0 else 0.0
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# Part 1: x >= 0 -> x
|
||||
# Part 2: -sqrt(3t) < x < 0 -> quadratic
|
||||
# Part 3: x <= -sqrt(3t) -> reciprocal
|
||||
|
||||
part1 = x
|
||||
part2 = self.coeff * x.pow(2) + x
|
||||
part3 = self.t / (x + 1e-9) # Add epsilon for stability
|
||||
|
||||
neg_res = torch.where(x > -self.sqrt_3t, part2, part3)
|
||||
|
||||
res = torch.where(x >= 0, part1, neg_res)
|
||||
|
||||
return res
|
||||
|
||||
|
||||
class Model(nn.Module):
|
||||
def __init__(self, t=1.0):
|
||||
super(Model, self).__init__()
|
||||
self.act = MMReLU(t=t)
|
||||
|
||||
def forward(self, x):
|
||||
return self.act(x)
|
||||
|
||||
def get_inputs():
|
||||
input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 2.0
|
||||
return [input_tensor.contiguous()]
|
||||
|
||||
def get_init_inputs():
|
||||
return [T_VALUE]
|
||||
|
|
@ -0,0 +1,74 @@
|
|||
###########################################################
|
||||
# 性能和精度验证程序
|
||||
###########################################################
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import time
|
||||
from MMReLU_torch import Model,get_inputs,get_init_inputs
|
||||
from MMReLU_cuda import ModelNew
|
||||
|
||||
def run_benchmark():
|
||||
# 检查 CUDA 是否可用
|
||||
if not torch.cuda.is_available():
|
||||
print("CUDA 不可用，请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。")
|
||||
return
|
||||
else:
|
||||
device = torch.device("cuda")
|
||||
|
||||
# 初始化模型
|
||||
init_inputs = get_init_inputs()
|
||||
init_inputs = [
|
||||
x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs
|
||||
]
|
||||
inputs = get_inputs()
|
||||
inputs = [
|
||||
x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs
|
||||
]
|
||||
|
||||
torch_model = Model(*init_inputs).cuda()
|
||||
cuda_model = ModelNew(*init_inputs).cuda()
|
||||
|
||||
torch_model.eval()
|
||||
cuda_model.eval()
|
||||
|
||||
print("-------------------- 精度对齐验证 --------------------")
|
||||
with torch.no_grad():
|
||||
output_torch = torch_model( *inputs)
|
||||
output_cuda = cuda_model(*inputs)
|
||||
|
||||
precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03)
|
||||
if precision_flag:
|
||||
print("✅ 精度对齐：两个模型的输出结果非常接近。")
|
||||
else:
|
||||
print("❌ 精度不一致！")
|
||||
|
||||
print("\n-------------------- 性能加速比测试 --------------------")
|
||||
num_iterations = 100
|
||||
|
||||
# PyTorch 模型计时
|
||||
torch.cuda.synchronize()
|
||||
start_time = time.time()
|
||||
for _ in range(num_iterations):
|
||||
_ = torch_model(*inputs)
|
||||
torch.cuda.synchronize()
|
||||
torch_time = (time.time() - start_time) / num_iterations
|
||||
|
||||
# 自定义 CUDA 内核计时
|
||||
torch.cuda.synchronize()
|
||||
start_time = time.time()
|
||||
for _ in range(num_iterations):
|
||||
_ = cuda_model(*inputs)
|
||||
torch.cuda.synchronize()
|
||||
cuda_time = (time.time() - start_time) / num_iterations
|
||||
|
||||
print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒")
|
||||
print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒")
|
||||
speedup = 0
|
||||
if cuda_time > 0:
|
||||
speedup = torch_time / cuda_time
|
||||
print(f"加速比 (Speedup): {speedup:.2f}x")
|
||||
else:
|
||||
print("CUDA 内核执行时间为0，无法计算加速比。")
|
||||
return precision_flag,speedup
|
||||
if __name__ == "__main__":
|
||||
precision_flag,speedup = run_benchmark()
|
||||
Loading…
Reference in New Issue