import torch
import torch.nn as nn
import math

BATCH_SIZE = 4096
HIDDEN_DIM = 4096
SHAPE = (BATCH_SIZE, HIDDEN_DIM)

# DSReLU 超参数 ,论文推荐 a=tan(85 deg), b=tan(10 deg)
A_VAL = math.tan(math.radians(85.0)) # approx 11.43
B_VAL = math.tan(math.radians(10.0)) # approx 0.176
K_VAL = 5.0

class DSReLU(nn.Module):
    '''
    DSReLU: A Novel Dynamic Slope Function for Superior Model Training
    https://arxiv.org/pdf/2408.09156
    Formula (Inference, t=1):
      f(x) = x            if x >= 0
      f(x) = x * s_final  if x < 0
    where s_final is a constant slope calculated from hyperparameters a, b, k:
      s_final = a + (b - a) / (1 + exp(-0.5 * k))s
    '''
    def __init__(self, a, b, k):
        super(DSReLU, self).__init__()
        # Inference, t=1
        t = 1.0
        self.slope = a + (b - a) / (1.0 + math.exp(-k * (t - 0.5)))

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        return torch.where(x >= 0, x, x * self.slope)

class Model(nn.Module):
    def __init__(self, a, b, k):
        super(Model, self).__init__()
        self.act = DSReLU(a, b, k)
    
    def forward(self, x):
        return self.act(x)

def get_inputs():
    input_tensor = torch.randn(SHAPE, dtype=torch.float32) * 2.0
    return [input_tensor.contiguous()]

def get_init_inputs():
    return [A_VAL, B_VAL, K_VAL]