forked from ccf-ai-infra/GPUCodeForces
50 lines
1.4 KiB
Python
50 lines
1.4 KiB
Python
import torch
|
|
import torch.nn as nn
|
|
|
|
class RMSNormTorchModel(nn.Module):
|
|
"""PyTorch原生RMSNorm实现"""
|
|
|
|
def __init__(self, hidden_size=256, eps=1e-6):
|
|
super(RMSNormTorchModel, self).__init__()
|
|
self.hidden_size = hidden_size
|
|
self.eps = eps
|
|
self.weight = nn.Parameter(torch.ones(hidden_size))
|
|
|
|
def forward(self, x):
|
|
# 计算均方根
|
|
variance = x.pow(2).mean(-1, keepdim=True)
|
|
# 归一化
|
|
x = x * torch.rsqrt(variance + self.eps)
|
|
# 应用权重
|
|
return self.weight * x
|
|
|
|
def get_init_inputs():
|
|
"""获取模型初始化参数"""
|
|
return [256] # hidden_size
|
|
|
|
def get_inputs():
|
|
"""获取模型输入数据"""
|
|
torch.manual_seed(42)
|
|
return [torch.randn(128, 256)]
|
|
|
|
def test_rmsnorm():
|
|
"""测试RMSNorm算子"""
|
|
# 创建测试数据
|
|
batch_size, hidden_size = 128, 256
|
|
input_tensor = torch.randn(batch_size, hidden_size, device='cuda' if torch.cuda.is_available() else 'cpu')
|
|
|
|
# 创建模型
|
|
torch_model = RMSNormTorchModel(hidden_size)
|
|
|
|
# 运行测试
|
|
with torch.no_grad():
|
|
torch_output = torch_model(input_tensor)
|
|
|
|
print(f"输入形状: {input_tensor.shape}")
|
|
print(f"输出形状: {torch_output.shape}")
|
|
print(f"输出范围: [{torch_output.min().item():.3f}, {torch_output.max().item():.3f}]")
|
|
|
|
return torch_output
|
|
|
|
if __name__ == "__main__":
|
|
test_rmsnorm() |