import torch
import torch.nn as nn

BATCH_SIZE = 4096
HIDDEN_DIM = 4096
SHAPE = (BATCH_SIZE, HIDDEN_DIM)

ALPHA_VALUE = 1.0

class ISRLU(nn.Module):
    """
     "ISRLU AND NISRLU: A NOVEL ACTIVATION FUNCTION AND A NOVEL INITIALIZER" (arXiv, 2017)
     Formula:
      f(x) = x                     if x >= 0
      f(x) = x * (1 / sqrt(1 + alpha * x^2)) if x < 0
    """
    def __init__(self, alpha=1.0):
        super(ISRLU, self).__init__()
        self.alpha = alpha

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        neg_part = x / torch.sqrt(1.0 + self.alpha * torch.pow(x, 2))
        return torch.where(x < 0, neg_part, x)

class Model(nn.Module):
    def __init__(self, alpha=1.0):
        super(Model, self).__init__()
        self.act = ISRLU(alpha)
    
    def forward(self, x):
        return self.act(x)

def get_inputs():
    input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 5.0
    return [input_tensor.contiguous()]

def get_init_inputs():
    return [ALPHA_VALUE]