Use linearly falling learning rate
authorMartin Pitt <martin@piware.de>
Sun, 30 Aug 2020 08:19:10 +0000 (10:19 +0200)
committerMartin Pitt <martin@piware.de>
Sun, 30 Aug 2020 09:40:28 +0000 (11:40 +0200)
This makes big strides on the completely untrained network, but treads
more carefully later on. Prevents overshooting with reLU, and even
slightly improves Sigmoid.

train.py

index 8510624..b294ce1 100755 (executable)
--- a/train.py
+++ b/train.py
@@ -28,7 +28,7 @@ print(f'correctly recognized images after initialization: {nnet.accuracy(net, te
 train_y = nnet.label_vectors(train_labels, 10)
 for i in range(100):
     for batch in range(0, num_train, nnet_batch):
-        cost = nnet.train(net, train_images[:, batch:(batch + nnet_batch)], train_y[:, batch:(batch + nnet_batch)], learning_rate=1)
+        cost = nnet.train(net, train_images[:, batch:(batch + nnet_batch)], train_y[:, batch:(batch + nnet_batch)], learning_rate=(100-i)/100)
     print(f'cost after training round {i}: {cost}')
 print(f'correctly recognized images after training: {nnet.accuracy(net, test_images, test_labels)}%')