forked from ccf-ai-infra/GPUCodeForces
32 lines
892 B
Python
32 lines
892 B
Python
import torch
|
|
import torch.nn as nn
|
|
|
|
class Model(nn.Module):
|
|
def __init__(self, scale: torch.Tensor, bias: torch.Tensor, temp: float):
|
|
super(Model, self).__init__()
|
|
self.scale = nn.Parameter(scale)
|
|
self.bias = nn.Parameter(bias)
|
|
self.temp = nn.Parameter(torch.tensor(float(temp), dtype=torch.float32))
|
|
|
|
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
z = x * self.scale.view(1,-1,1,1) + self.bias.view(1,-1,1,1)
|
|
t = self.temp
|
|
zt = z / t
|
|
m = torch.amax(zt, dim=1, keepdim=True)
|
|
ex = torch.exp(zt - m)
|
|
den = torch.sum(ex, dim=1, keepdim=True)
|
|
g = ex / den
|
|
return g * x
|
|
|
|
N, C, H, W = 4, 128, 32, 32
|
|
|
|
def get_inputs():
|
|
x = torch.randn(N, C, H, W)
|
|
return [x]
|
|
|
|
def get_init_inputs():
|
|
scale = torch.randn(C)
|
|
bias = torch.randn(C)
|
|
temp = 0.8
|
|
return [scale, bias, temp]
|