Tweak network structure

Increase first layer size, otherwise the network does not "see" enough
detail in the input.  Drop second layer, it makes the recognition rate
actually much worse.

diff --git a/README.md b/README.md
index 3af0686d6972af783a60d8ca377d150c54e01cad..d48ebf3bc0fd713a836bbb238992941d5504f7aa 100644 (file)
--- a/README.md
+++ b/README.md
@@ -72,10 +72,10 @@ sys 1m10.169s
  - This is way too slow. I found an [interesting approach](https://www.kdnuggets.com/2019/08/numpy-neural-networks-computational-graphs.html) that harnesses the power of numpy by doing the computations for lots of images in parallel, instead of spending a lot of time in Python on iterating over tens of thousands of examples. Now the accuracy computation takes only negligible time instead of 6 seconds, and each round of training takes less than a second:
 ```
 $ time ./train.py
  - This is way too slow. I found an [interesting approach](https://www.kdnuggets.com/2019/08/numpy-neural-networks-computational-graphs.html) that harnesses the power of numpy by doing the computations for lots of images in parallel, instead of spending a lot of time in Python on iterating over tens of thousands of examples. Now the accuracy computation takes only negligible time instead of 6 seconds, and each round of training takes less than a second:
 ```
 $ time ./train.py

-output vector of first image: [0.50863223 0.50183558 0.50357349 0.50056673 0.50285531 0.5043152
- 0.51588292 0.49403    0.5030618  0.51006963]
-classification of first image: 6 with confidence 0.5158829224337754; real label 7
-correctly recognized images after initialization: 9.58%
+output vector of first image: [0.51452796 0.49736819 0.51415083 0.50027547 0.48447025 0.49759904
+ 0.52621162 0.48671402 0.517606   0.50214569]
+classification of first image: 6 with confidence 0.526211616929459; real label 7
+correctly recognized images after initialization: 7.75%

 cost after training round 0: 1.0462266880961681
 [...]
 cost after training round 99: 0.4499245817840479
 cost after training round 0: 1.0462266880961681
 [...]
 cost after training round 99: 0.4499245817840479


@@ -85,3 +85,30 @@ real 1m51.520s
 user   4m23.863s
 sys    2m31.686s
 ```
 user   4m23.863s
 sys    2m31.686s
 ```

+
+- Poor recognition quality after 100 iterations, as the network structure is apparently inappropriate. Having only 16 neurons in the first hidden layer makes the network not able to "see enough details" in the input. So let's use 128 neurons in the first hidden layer, and drop the second layer (it only seems to make things worse for me). Naturally a single training round takes much longer now, but voilà, after only 20 learning iterations it's already quite respectable:
+```
+$ time ./train.py
+correctly recognized images after initialization: 9.8%
+cost after training round 0: 0.44518003660592853
+[...]
+cost after training round 19: 0.10783488337150668
+correctly recognized images after training: 89.09%
+
+real   0m47.603s
+user   2m12.141s
+```
+
+ - And after 100 iterations, the accuracy improves even more, and the classification of the first test image looks reasonable:
+```
+cost after training round 99: 0.043068345296584126
+correctly recognized images after training: 94.17%
+
+output vector of first image: [1.11064478e-02 5.59058012e-03 5.40483856e-02 7.93664914e-02
+ 2.22662031e-03 3.50355065e-03 2.57506703e-04 9.60761429e-01
+ 2.68869803e-03 5.26559410e-03]
+
+
+real   4m10.904s
+user   11m21.203s
+```

diff --git a/train.py b/train.py
index 012e2c7f3527c6917d86fdfa2392b44c12f81d78..5ff76669709f1d2a9549dd25b1f16a7b4c97f632 100755 (executable)
--- a/train.py
+++ b/train.py
@@ -13,15 +13,13 @@ nnet_batch = 10000
 # neural network structure: two hidden layers, one output layer
 #                   (input)--> [Linear->Sigmoid] -> [Linear->Sigmoid] -->(output)
 # handle 10,000 vectors at a time
 # neural network structure: two hidden layers, one output layer
 #                   (input)--> [Linear->Sigmoid] -> [Linear->Sigmoid] -->(output)
 # handle 10,000 vectors at a time

-Z1 = nnet.LinearLayer(input_shape=(rows * cols, nnet_batch), n_out=20)
+Z1 = nnet.LinearLayer(input_shape=(rows * cols, nnet_batch), n_out=80)

 A1 = nnet.SigmoidLayer(Z1.Z.shape)
 A1 = nnet.SigmoidLayer(Z1.Z.shape)

-Z2 = nnet.LinearLayer(input_shape=A1.A.shape, n_out=16)
-A2 = nnet.SigmoidLayer(Z2.Z.shape)
-ZO = nnet.LinearLayer(input_shape=A2.A.shape, n_out=10)
+ZO = nnet.LinearLayer(input_shape=A1.A.shape, n_out=10)

 AO = nnet.SigmoidLayer(ZO.Z.shape)
 AO = nnet.SigmoidLayer(ZO.Z.shape)

-net = (Z1, A1, Z2, A2, ZO, AO)
+net = (Z1, A1, ZO, AO)

 
 

-res = nnet.forward(net, train_images[:, 0:10000])
+res = nnet.forward(net, test_images[:, 0:10000])

 print(f'output vector of first image: {res[:, 0]}')
 digit, conf = nnet.classify(res[:, 0])
 print(f'classification of first image: {digit} with confidence {conf}; real label {test_labels[0]}')
 print(f'output vector of first image: {res[:, 0]}')
 digit, conf = nnet.classify(res[:, 0])
 print(f'classification of first image: {digit} with confidence {conf}; real label {test_labels[0]}')


@@ -30,6 +28,11 @@ print(f'correctly recognized images after initialization: {nnet.accuracy(net, te
 train_y = nnet.label_vectors(train_labels, 10)
 for i in range(100):
     for batch in range(0, num_train, nnet_batch):
 train_y = nnet.label_vectors(train_labels, 10)
 for i in range(100):
     for batch in range(0, num_train, nnet_batch):

-        cost = nnet.train(net, train_images[:, batch:(batch + nnet_batch)], train_y[:, batch:(batch + nnet_batch)])
+        cost = nnet.train(net, train_images[:, batch:(batch + nnet_batch)], train_y[:, batch:(batch + nnet_batch)], learning_rate=1)

     print(f'cost after training round {i}: {cost}')
 print(f'correctly recognized images after training: {nnet.accuracy(net, test_images, test_labels)}%')
     print(f'cost after training round {i}: {cost}')
 print(f'correctly recognized images after training: {nnet.accuracy(net, test_images, test_labels)}%')

+
+res = nnet.forward(net, test_images[:, 0:10000])
+print(f'output vector of first image: {res[:, 0]}')
+digit, conf = nnet.classify(res[:, 0])
+print(f'classification of first image: {digit} with confidence {conf}; real label {test_labels[0]}')