X-Git-Url: https://piware.de/gitweb/?p=handwriting-recognition.git;a=blobdiff_plain;f=train.py;h=b29effaa799aeb738f932c8e3260facdf6467a86;hp=8a6dc96f2c1d40281d452fc3c894fffb79a73436;hb=8dcd00e9f8bbfc569c9b29ac06d748320d8bf737;hpb=0ea12b213873b4bef12e1f2b65eed64704ee040f diff --git a/train.py b/train.py index 8a6dc96..b29effa 100755 --- a/train.py +++ b/train.py @@ -46,7 +46,7 @@ for i in range(1, NUM_LAYERS): biases.append(rg.normal(scale=10, size=SIZES[i])) -def feed_forward(x, transfer=reLU): +def feed_forward(x, transfer=sigmoid): '''Compute all z and output vectors for given input vector''' a_s = [x] @@ -63,6 +63,59 @@ def classify(y): return np.argmax(y), np.max(y) +def cost_grad(x, target_y, transfer=sigmoid, transfer_prime=sigmoid_prime): + '''Return (∂C/∂w, ∂C/∂b) for a particular input and desired output vector''' + + # forward pass, remember all z vectors and activations for every layer + z_s, a_s = feed_forward(x, transfer) + + # backward pass + deltas = [None] * len(weights) # delta = dC/dz error for each layer + # insert the last layer error + deltas[-1] = transfer_prime(z_s[-1]) * 2 * (a_s[-1] - target_y) + for i in reversed(range(len(deltas) - 1)): + deltas[i] = (weights[i + 1].T @ deltas[i + 1]) * transfer_prime(z_s[i]) + + dw = [d @ a_s[i+1] for i, d in enumerate(deltas)] + db = deltas + return dw, db + + +def label_vector(label): + x = np.zeros(10) + x[label] = 1.0 + return x + + +def backpropagate(image_batch, label_batch, eta): + '''Update NN with gradient descent and backpropagation to a batch of inputs + + eta is the learning rate. + ''' + global weights, biases + + num_images = image_batch.shape[1] + for i in range(num_images): + y = label_vector(label_batch[i]) + dws, dbs = cost_grad(image_batch[:, i], y) + weights = [w + eta * dw for w, dw in zip(weights, dws)] + biases = [b + eta * db for b, db in zip(biases, dbs)] + + +def train(images, labels, eta, batch_size=100): + '''Do backpropagation for smaller batches + + This greatly speeds up the learning process, at the expense of finding a more erratic path to the local minimum. + ''' + num_images = images.shape[1] + offset = 0 + while offset < num_images: + images_batch = images[:, offset:offset + batch_size] + labels_batch = labels[offset:offset + batch_size] + backpropagate(images_batch, labels_batch, eta) + offset += batch_size + + def test(): """Count percentage of test inputs which are being recognized correctly""" @@ -80,3 +133,9 @@ print(f'output vector of first image: {res[1][-1]}') digit, conf = classify(res[1][-1]) print(f'classification of first image: {digit} with confidence {conf}; real label {test_labels[0]}') print(f'correctly recognized images after initialization: {test()}%') + +for i in range(1): + print(f"round #{i} of learning...") + train(test_images, test_labels, 1) + +print(f'correctly recognized images: {test()}%')