From 91e5c285486f75219a48cd55e08e346bcb7f08e3 Mon Sep 17 00:00:00 2001
From: sql-hkr <sql.hkr@gmail.com>
Date: Sat, 25 Oct 2025 23:23:11 +0900
Subject: [PATCH] Add LayerNorm (#20)

Introduced a new LayerNorm class in syna.layers.normalization, implementing layer normalization as described by Ba et al. (2016). Updated __init__.py to export LayerNorm.
---
 src/syna/core.py                 |  2 +-
 src/syna/layers/__init__.py      |  1 +
 src/syna/layers/normalization.py | 39 ++++++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 1 deletion(-)
 create mode 100644 src/syna/layers/normalization.py

diff --git a/src/syna/core.py b/src/syna/core.py
index c4ce0ba..1ad2bd7 100644
--- a/src/syna/core.py
+++ b/src/syna/core.py
@@ -439,7 +439,7 @@ class Function:
     """
 
     def __call__(self, *input: Tensor | np.ndarray | int | float) -> Any:
-        inputs = [as_tensor(x) for x in input]
+        inputs = [as_tensor(x) if x is not None else Tensor(x) for x in input]
         xs = [x.data for x in inputs]
         ys = self.forward(*xs)
         if not isinstance(ys, tuple):
diff --git a/src/syna/layers/__init__.py b/src/syna/layers/__init__.py
index 9c878a2..fcc9072 100644
--- a/src/syna/layers/__init__.py
+++ b/src/syna/layers/__init__.py
@@ -1,2 +1,3 @@
 from syna.layers.layer import Layer, Linear, Parameter
+from syna.layers.normalization import LayerNorm
 from syna.layers.rnn import LSTM, RNN
diff --git a/src/syna/layers/normalization.py b/src/syna/layers/normalization.py
new file mode 100644
index 0000000..1d8e771
--- /dev/null
+++ b/src/syna/layers/normalization.py
@@ -0,0 +1,39 @@
+"""Normalization layers"""
+
+from __future__ import annotations
+
+import numpy as np
+
+import syna.functions as F
+from syna.core import Parameter
+from syna.layers.layer import Layer
+
+
+class LayerNorm(Layer):
+    r"""Layer Normalization (Jimmy Lei Ba et al. 2016).
+
+    paper: https://arxiv.org/abs/1607.06450
+
+    .. math::
+        y = \gamma \odot \frac{x - \mathbb{E}[x]}{ \sqrt{\mathrm{Var}[x] + \epsilon}} + \beta
+
+    Args:
+        dim: size of the last (normalized) dimension
+        eps: small constant added to variance for numerical stability
+    """
+
+    def __init__(self, dim: int, eps: float = 1e-5, dtype=np.float32) -> None:
+        super().__init__()
+        self.eps = eps
+        self.gamma = Parameter(np.ones(dim, dtype=dtype), name="gamma")
+        self.beta = Parameter(np.zeros(dim, dtype=dtype), name="beta")
+
+    def forward(self, x):
+        """Apply layer normalization to input tensor."""
+        axis = len(x.shape) - 1
+        mean = F.mean(x, axis=axis, keepdims=True)
+        diff = x - mean
+        var = F.mean(diff * diff, axis=axis, keepdims=True)
+        std = F.sqrt(var + self.eps)
+        x_hat = diff / std
+        return x_hat * self.gamma + self.beta