import torch
import torch.nn as nn

BATCH_SIZE = 4096
HIDDEN_DIM = 4096
SHAPE = (BATCH_SIZE, HIDDEN_DIM)

class Nish(nn.Module):
    """
    Nish Activation.
    https://arxiv.org/abs/2210.09083v1
    Formula:
      f(x) = x                               if x >= 0
      f(x) = sigmoid(x) * (x + sin(x))       if x < 0
    """
    def __init__(self):
        super(Nish, self).__init__()

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        pos_part = x
        neg_part = torch.sigmoid(x) * (x + torch.sin(x))
        return torch.where(x >= 0, pos_part, neg_part)

class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.act = Nish()
    
    def forward(self, x):
        return self.act(x)

def get_inputs():
    input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 5.0
    return [input_tensor.contiguous()]

def get_init_inputs():
    return []