import torch
import torch.nn as nn

BATCH_SIZE = 4096
HIDDEN_DIM = 4096
SHAPE = (BATCH_SIZE, HIDDEN_DIM)

BETA_VAL = 0.5

class BLU(nn.Module):
    '''
    Bendable Linear Units
    L. B. Godfrey, “An evaluation of parametric activation functions for deep learning,” in Proc. IEEE Int. Conf. Syst., Man Cybern. (SMC), Oct. 2019, pp. 3006–3011.
    
    Formula: f(x) = beta * (sqrt(x^2 + 1) - 1) + x
    '''
    def __init__(self, beta=0.5):
        super(BLU, self).__init__()
        self.beta = beta

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return self.beta * (torch.sqrt(x.pow(2) + 1.0) - 1.0) + x

class Model(nn.Module):
    def __init__(self, beta=0.5):
        super(Model, self).__init__()
        self.act = BLU(beta=beta)
    
    def forward(self, x):
        return self.act(x)

def get_inputs():
    input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 5.0
    return [input_tensor.contiguous()]

def get_init_inputs():
    return [BETA_VAL]