X-Git-Url: https://piware.de/gitweb/?p=handwriting-recognition.git;a=blobdiff_plain;f=nnet.py;h=239c96ba24bdccb491b165044fb25f1885f8785f;hp=ddf8ba68b948bfda759c4d8b5696f0efd6c39559;hb=d986cacf7fc94fb78904f01e11128d666efff804;hpb=0b40285a04bfbf2d73f7a7154eacb4613f08b350

diff --git a/nnet.py b/nnet.py
index ddf8ba6..239c96b 100644
--- a/nnet.py
+++ b/nnet.py
@@ -59,6 +59,20 @@ class SigmoidLayer:
         return upstream_grad * self.A * (1 - self.A)
 
 
+class reLULayer:
+    def __init__(self, shape):
+        self.shape = shape
+
+    def forward(self, Z):
+        assert Z.shape == self.shape
+        self.A = np.maximum(Z, 0)
+        return self.A
+
+    def backward(self, upstream_grad, learning_rate=0.1):
+        # couple upstream gradient with local gradient, the result will be sent back to the Linear layer
+        return upstream_grad * np.heaviside(self.A, 1)
+
+
 def label_vectors(labels, n):
     y = np.zeros((n, labels.size))
     for i, l in enumerate(labels):