Add reLU layer implementation

[handwriting-recognition.git] / nnet.py

diff --git a/nnet.py b/nnet.py
index ddf8ba68b948bfda759c4d8b5696f0efd6c39559..239c96ba24bdccb491b165044fb25f1885f8785f 100644 (file)
--- a/nnet.py
+++ b/nnet.py
@@ -59,6 +59,20 @@ class SigmoidLayer:
         return upstream_grad * self.A * (1 - self.A)
 
 
+class reLULayer:
+    def __init__(self, shape):
+        self.shape = shape
+
+    def forward(self, Z):
+        assert Z.shape == self.shape
+        self.A = np.maximum(Z, 0)
+        return self.A
+
+    def backward(self, upstream_grad, learning_rate=0.1):
+        # couple upstream gradient with local gradient, the result will be sent back to the Linear layer
+        return upstream_grad * np.heaviside(self.A, 1)
+
+
 def label_vectors(labels, n):
     y = np.zeros((n, labels.size))
     for i, l in enumerate(labels):