forked from ccf-ai-infra/GPUCodeForces
24 lines
689 B
Python
24 lines
689 B
Python
import torch
|
|
import torch.nn as nn
|
|
import torch.nn.functional as F
|
|
|
|
class Model(nn.Module):
|
|
def __init__(self, in_features):
|
|
super().__init__()
|
|
torch.manual_seed(42)
|
|
self.linear1 = nn.Linear(in_features, in_features) # 特征变换
|
|
self.linear2 = nn.Linear(in_features, in_features) # 门控变换
|
|
|
|
def forward(self, x):
|
|
x1 = self.linear1(x) # 主分支
|
|
x2 = self.linear2(x) # 门控分支
|
|
return x1 * torch.sigmoid(1.702 * x2) # SwiGLU公式
|
|
|
|
def get_inputs():
|
|
batch_size = 512
|
|
in_features = 512
|
|
x = torch.randn(batch_size, in_features)
|
|
return [x]
|
|
|
|
def get_init_inputs():
|
|
return [512] # in_features |