From d3310633438333313ad128e45bb70454c1b8355b Mon Sep 17 00:00:00 2001 From: hli28146 Date: Thu, 4 Dec 2025 16:32:30 +0800 Subject: [PATCH] finish fake_quantize_per_tensor_affine #38 --- .../fake_quantize_per_tensor_affine_cuda.py | 148 ++++++++++++++++++ .../fake_quantize_per_tensor_affine_torch.py | 41 +++++ S1/hli28146_#38/prompt.txt | 64 ++++++++ S1/hli28146_#38/run_code.py | 74 +++++++++ 4 files changed, 327 insertions(+) create mode 100644 S1/hli28146_#38/fake_quantize_per_tensor_affine_cuda.py create mode 100644 S1/hli28146_#38/fake_quantize_per_tensor_affine_torch.py create mode 100644 S1/hli28146_#38/prompt.txt create mode 100644 S1/hli28146_#38/run_code.py diff --git a/S1/hli28146_#38/fake_quantize_per_tensor_affine_cuda.py b/S1/hli28146_#38/fake_quantize_per_tensor_affine_cuda.py new file mode 100644 index 00000000..0aec63e6 --- /dev/null +++ b/S1/hli28146_#38/fake_quantize_per_tensor_affine_cuda.py @@ -0,0 +1,148 @@ +import torch +import torch.nn as nn +from torch.utils.cpp_extension import load_inline + +cpp_source = """ +#include + +torch::Tensor fake_quantize_cuda_forward( + const torch::Tensor& input, + const torch::Tensor& scale, + const torch::Tensor& zero_point, + int64_t quant_min, + int64_t quant_max); +""" + +cuda_source = """ +#include +#include +#include + +struct __align__(16) Float4 { + float x, y, z, w; +}; + +// Core logic based on user request: +// q = clamp(round(x / s) + z, qmin, qmax) +// out = (q - z) * s +__device__ __forceinline__ float fake_quant_op( + float x, float s, int z, int qmin, int qmax) +{ + // 1. Division + float val = x / s; + + // 2. Rounding (rintf rounds to nearest integer, ties to even) + // Formula correction: round(x/s) then add z + val = rintf(val); + + // 3. Add Zero Point + val += (float)z; + + // 4. Clamp + val = fmaxf(val, (float)qmin); + val = fminf(val, (float)qmax); + + // 5. Dequantize + return (val - (float)z) * s; +} + +__global__ void fake_quantize_kernel( + const float* __restrict__ input, + float* __restrict__ output, + const float* __restrict__ scale_ptr, + const int* __restrict__ zero_point_ptr, + const int qmin, + const int qmax, + const int n_elements) +{ + // Load scalars once per thread/block (cached) + float s = *scale_ptr; + int z = *zero_point_ptr; + + int idx = blockIdx.x * blockDim.x + threadIdx.x; + int stride = blockDim.x * gridDim.x; + + // 1. Vectorized Loop + int vec_loops = n_elements / 4; + const Float4* vec_input = reinterpret_cast(input); + Float4* vec_output = reinterpret_cast(output); + + for (int i = idx; i < vec_loops; i += stride) { + Float4 in_val = vec_input[i]; + Float4 out_val; + + out_val.x = fake_quant_op(in_val.x, s, z, qmin, qmax); + out_val.y = fake_quant_op(in_val.y, s, z, qmin, qmax); + out_val.z = fake_quant_op(in_val.z, s, z, qmin, qmax); + out_val.w = fake_quant_op(in_val.w, s, z, qmin, qmax); + + vec_output[i] = out_val; + } + + // 2. Tail Loop + int tail_start = vec_loops * 4; + for (int i = tail_start + idx; i < n_elements; i += stride) { + output[i] = fake_quant_op(input[i], s, z, qmin, qmax); + } +} + +torch::Tensor fake_quantize_cuda_forward( + const torch::Tensor& input, + const torch::Tensor& scale, + const torch::Tensor& zero_point, + int64_t quant_min, + int64_t quant_max) +{ + TORCH_CHECK(input.is_cuda(), "Input must be CUDA"); + TORCH_CHECK(input.is_contiguous(), "Input must be contiguous"); + TORCH_CHECK(scale.is_cuda(), "Scale must be CUDA"); + TORCH_CHECK(zero_point.is_cuda(), "Zero Point must be CUDA"); + + auto output = torch::empty_like(input); + int n_elements = input.numel(); + + const int block_size = 256; + int grid_size = (n_elements + block_size * 4 - 1) / (block_size * 4); + if (grid_size > 65535) grid_size = 65535; + + fake_quantize_kernel<<>>( + input.data_ptr(), + output.data_ptr(), + scale.data_ptr(), + zero_point.data_ptr(), + (int)quant_min, + (int)quant_max, + n_elements + ); + + return output; +} +""" + +fake_quant_op = load_inline( + name='fake_quant_op', + cpp_sources=cpp_source, + cuda_sources=cuda_source, + functions=['fake_quantize_cuda_forward'], + verbose=False, + extra_cuda_cflags=['-O3', '--use_fast_math'] +) + +class FakeQuantizeNew(nn.Module): + def __init__(self, quant_min, quant_max): + super(FakeQuantizeNew, self).__init__() + self.quant_min = quant_min + self.quant_max = quant_max + + def forward(self, x, scale, zero_point): + return fake_quant_op.fake_quantize_cuda_forward( + x, scale, zero_point, self.quant_min, self.quant_max + ) + +class ModelNew(nn.Module): + def __init__(self): + super(ModelNew, self).__init__() + self.fq = FakeQuantizeNew(-128, 127) + + def forward(self, x, scale, zero_point): + return self.fq(x, scale, zero_point) \ No newline at end of file diff --git a/S1/hli28146_#38/fake_quantize_per_tensor_affine_torch.py b/S1/hli28146_#38/fake_quantize_per_tensor_affine_torch.py new file mode 100644 index 00000000..a3df4832 --- /dev/null +++ b/S1/hli28146_#38/fake_quantize_per_tensor_affine_torch.py @@ -0,0 +1,41 @@ +import torch +import torch.nn as nn + +BATCH_SIZE = 2048 +DIM = 2048 +SHAPE = (BATCH_SIZE, DIM) + +QMIN = -128 +QMAX = 127 + +class FakeQuantize(nn.Module): + """ + Fake Quantize Per Tensor Affine + """ + def __init__(self, quant_min=QMIN, quant_max=QMAX): + super(FakeQuantize, self).__init__() + self.quant_min = quant_min + self.quant_max = quant_max + + def forward(self, x, scale, zero_point): + return torch.fake_quantize_per_tensor_affine( + x, scale, zero_point, self.quant_min, self.quant_max + ) + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.fq = FakeQuantize() + + def forward(self, x, scale, zero_point): + return self.fq(x, scale, zero_point) + +def get_inputs(): + x = torch.randn(SHAPE, dtype=torch.float32) + scale = torch.tensor([0.05], dtype=torch.float32) + zero_point = torch.tensor([10], dtype=torch.int32) + + return [x.contiguous(), scale, zero_point] + +def get_init_inputs(): + return [] \ No newline at end of file diff --git a/S1/hli28146_#38/prompt.txt b/S1/hli28146_#38/prompt.txt new file mode 100644 index 00000000..5b5ed5fe --- /dev/null +++ b/S1/hli28146_#38/prompt.txt @@ -0,0 +1,64 @@ +Write a custom CUDA kernel to optimize the Fake Quantize operation based on a specific formula. + +The mathematical definition provided is: +output = (clamp(round(input / scale) + zero_point, quant_min, quant_max) - zero_point) * scale + +Inputs: +- input: Float32 Tensor. +- scale: Scalar Float32. +- zero_point: Scalar Int32. +- quant_min, quant_max: Scalar Int32. + +Problem Analysis: +This operation simulates quantization error. It involves element-wise division, rounding, addition, clamping, subtraction, and multiplication. Doing this naively involves high memory bandwidth usage. + +Optimization Strategy: +1. **Fused Kernel**: Perform the entire logic in a single CUDA kernel pass. +2. **Specific Rounding Logic**: Implement `round(input/scale) + zero_point` (rounding before adding zero_point) as requested. Use `rintf` for "round to nearest even". +3. **Vectorized Access**: Use `float4` loads/stores to process 4 elements per thread, maximizing memory bandwidth. +4. **Scalar Optimization**: Load `scale` and `zero_point` once per thread/block from global memory and keep them in registers. + +Here's an example to show you the syntax of inline embedding custom CUDA operators in torch: The example given architecture is: + +```python +import torch +import torch.nn as nn + +BATCH_SIZE = 2048 +DIM = 2048 +SHAPE = (BATCH_SIZE, DIM) + +QMIN = -128 +QMAX = 127 + +class FakeQuantize(nn.Module): + """ + Fake Quantize Per Tensor Affine + """ + def __init__(self, quant_min=QMIN, quant_max=QMAX): + super(FakeQuantize, self).__init__() + self.quant_min = quant_min + self.quant_max = quant_max + + def forward(self, x, scale, zero_point): + return torch.fake_quantize_per_tensor_affine( + x, scale, zero_point, self.quant_min, self.quant_max + ) + +class Model(nn.Module): + def __init__(self): + super(Model, self).__init__() + self.fq = FakeQuantize() + + def forward(self, x, scale, zero_point): + return self.fq(x, scale, zero_point) + +def get_inputs(): + x = torch.randn(SHAPE, dtype=torch.float32) + scale = torch.tensor([0.05], dtype=torch.float32) + zero_point = torch.tensor([10], dtype=torch.int32) + + return [x.contiguous(), scale, zero_point] + +def get_init_inputs(): + return [] \ No newline at end of file diff --git a/S1/hli28146_#38/run_code.py b/S1/hli28146_#38/run_code.py new file mode 100644 index 00000000..efca0511 --- /dev/null +++ b/S1/hli28146_#38/run_code.py @@ -0,0 +1,74 @@ +########################################################### +# 性能和精度验证程序 +########################################################### +import torch +import torch.nn as nn +import time +from fake_quantize_per_tensor_affine_torch import Model,get_inputs,get_init_inputs +from fake_quantize_per_tensor_affine_cuda import ModelNew + +def run_benchmark(): + # 检查 CUDA 是否可用 + if not torch.cuda.is_available(): + print("CUDA 不可用，请确保您有可用的 NVIDIA GPU 并已正确安装 PyTorch CUDA 版本。") + return + else: + device = torch.device("cuda") + + # 初始化模型 + init_inputs = get_init_inputs() + init_inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in init_inputs + ] + inputs = get_inputs() + inputs = [ + x.cuda(device=device) if isinstance(x, torch.Tensor) else x for x in inputs + ] + + torch_model = Model(*init_inputs).cuda() + cuda_model = ModelNew(*init_inputs).cuda() + + torch_model.eval() + cuda_model.eval() + + print("-------------------- 精度对齐验证 --------------------") + with torch.no_grad(): + output_torch = torch_model( *inputs) + output_cuda = cuda_model(*inputs) + + precision_flag = torch.allclose(output_torch, output_cuda,rtol=1e-03) + if precision_flag: + print("✅ 精度对齐：两个模型的输出结果非常接近。") + else: + print("❌ 精度不一致！") + + print("\n-------------------- 性能加速比测试 --------------------") + num_iterations = 100 + + # PyTorch 模型计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = torch_model(*inputs) + torch.cuda.synchronize() + torch_time = (time.time() - start_time) / num_iterations + + # 自定义 CUDA 内核计时 + torch.cuda.synchronize() + start_time = time.time() + for _ in range(num_iterations): + _ = cuda_model(*inputs) + torch.cuda.synchronize() + cuda_time = (time.time() - start_time) / num_iterations + + print(f"PyTorch torch.relu 平均执行时间: {torch_time:.6f} 秒") + print(f"自定义 CUDA 内核 平均执行时间: {cuda_time:.6f} 秒") + speedup = 0 + if cuda_time > 0: + speedup = torch_time / cuda_time + print(f"加速比 (Speedup): {speedup:.2f}x") + else: + print("CUDA 内核执行时间为0，无法计算加速比。") + return precision_flag,speedup +if __name__ == "__main__": + precision_flag,speedup = run_benchmark() \ No newline at end of file