298 lines
7.3 KiB
Python
298 lines
7.3 KiB
Python
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
# Copyright (C) 2026 David (Corey) Herrera
|
|
"""
|
|
Comparison tests for activation functions.
|
|
"""
|
|
|
|
import numpy as np
|
|
import torch
|
|
import torch.nn.functional as F
|
|
from compare_utils import ComparisonTest, TestRunner
|
|
|
|
|
|
class TestReLU(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("relu", tolerance=0.0)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32),)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.relu(x)
|
|
|
|
|
|
class TestSigmoid(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("sigmoid", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 5,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return torch.sigmoid(x)
|
|
|
|
|
|
class TestTanh(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("tanh", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return torch.tanh(x)
|
|
|
|
|
|
class TestGELU(ComparisonTest):
|
|
def __init__(self):
|
|
# GELU uses different approximations - allow larger tolerance
|
|
super().__init__("gelu", tolerance=1e-3)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.gelu(x)
|
|
|
|
|
|
class TestSiLU(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("silu", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.silu(x)
|
|
|
|
|
|
class TestMish(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("mish", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.mish(x)
|
|
|
|
|
|
class TestSELU(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("selu", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.selu(x)
|
|
|
|
|
|
class TestCELU(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("celu", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.celu(x, alpha=1.0)
|
|
|
|
def foundry_params(self):
|
|
return {"alpha": 1.0}
|
|
|
|
|
|
class TestELU(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("elu", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.elu(x, alpha=1.0)
|
|
|
|
def foundry_params(self):
|
|
return {"alpha": 1.0}
|
|
|
|
|
|
class TestReLU6(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("relu6", tolerance=0.0)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 10,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.relu6(x)
|
|
|
|
|
|
class TestLeakyReLU(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("leaky_relu", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 5,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.leaky_relu(x, negative_slope=0.01)
|
|
|
|
def foundry_params(self):
|
|
return {"slope": 0.01}
|
|
|
|
|
|
class TestSoftmax(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("softmax", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(32, 100).astype(np.float32) * 2,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.softmax(x, dim=-1)
|
|
|
|
def foundry_params(self):
|
|
return {"dim": -1}
|
|
|
|
|
|
class TestLogSoftmax(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("log_softmax", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(32, 100).astype(np.float32) * 2,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.log_softmax(x, dim=-1)
|
|
|
|
def foundry_params(self):
|
|
return {"dim": -1}
|
|
|
|
|
|
class TestHardsigmoid(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("hardsigmoid", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 5,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.hardsigmoid(x)
|
|
|
|
|
|
class TestHardswish(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("hardswish", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 5,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.hardswish(x)
|
|
|
|
|
|
class TestHardtanh(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("hardtanh", tolerance=0.0)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.hardtanh(x, min_val=-1.0, max_val=1.0)
|
|
|
|
def foundry_params(self):
|
|
return {"min": -1.0, "max": 1.0}
|
|
|
|
|
|
class TestSoftsign(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("softsign", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 5,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.softsign(x)
|
|
|
|
|
|
class TestTanhshrink(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("tanhshrink", tolerance=1e-6)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 3,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.tanhshrink(x)
|
|
|
|
|
|
class TestSoftplus(ComparisonTest):
|
|
def __init__(self):
|
|
super().__init__("softplus", tolerance=1e-5)
|
|
|
|
def generate_input(self, seed=42):
|
|
np.random.seed(seed)
|
|
return (np.random.randn(1000).astype(np.float32) * 5,)
|
|
|
|
def pytorch_forward(self, x):
|
|
return F.softplus(x, beta=1.0, threshold=20.0)
|
|
|
|
def foundry_params(self):
|
|
return {"beta": 1.0, "threshold": 20.0}
|
|
|
|
|
|
def get_all_tests():
|
|
"""Return all activation tests."""
|
|
return [
|
|
TestReLU(),
|
|
TestSigmoid(),
|
|
TestTanh(),
|
|
TestGELU(),
|
|
TestSiLU(),
|
|
TestMish(),
|
|
TestSELU(),
|
|
TestCELU(),
|
|
TestELU(),
|
|
TestReLU6(),
|
|
TestLeakyReLU(),
|
|
TestSoftmax(),
|
|
TestLogSoftmax(),
|
|
TestHardsigmoid(),
|
|
TestHardswish(),
|
|
TestHardtanh(),
|
|
TestSoftsign(),
|
|
TestTanhshrink(),
|
|
TestSoftplus(),
|
|
]
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import sys
|
|
debug = "--debug" in sys.argv
|
|
|
|
runner = TestRunner()
|
|
for test in get_all_tests():
|
|
runner.add_test(test)
|
|
|
|
runner.run_all(debug=debug)
|
|
runner.save_results()
|