ffcm/nn-scratch-mnist

1import numpy as np2 3class NeuralNetwork():4    def __init__(self, neurons_per_layer):5        self.num_layers = len(neurons_per_layer)6        self.neurons_per_layer = neurons_per_layer7 8        a = neurons_per_layer[1:]9        b = neurons_per_layer[:-1]10 11        self.weights = [12            np.random.randn(current, previous) for current, previous in13            zip(a, b)14        ]15 16        self.bias = [np.random.randn(y, 1) for y in a]17 18    def activation_fn(self, x):19        return 1.0 / (1.0 + np.exp(-x))20 21    def cost_derivative(self, output, expected):22        return output - expected23 24    def activation_derivative(self, x):25        return self.activation_fn(x) * (1 - self.activation_fn(x))26 27    def feed_forward(self, x):28        for w, b in zip(self.weights, self.bias):29            z = np.dot(w, x) + b30            x = self.activation_fn(z)31 32        return x33 34    def backprop(self, x, expected):35        weight_gradients = [np.zeros(w.shape) for w in self.weights]36        bias_gradients = [np.zeros(b.shape) for b in self.bias]37 38        zs = []39        activation = np.array(x)40        activations = [np.array(x)]41 42        for w, b in zip(self.weights, self.bias):43            z = np.dot(w, activation) + b44            zs.append(z)45            activation = self.activation_fn(z)46            activations.append(activation)47 48        delta = self.cost_derivative(49            activation, expected) * self.activation_derivative(zs[-1])50 51        weight_gradients[-1] = np.dot(delta, activations[-2].T)52        bias_gradients[-1] = delta53 54        for layer in range(2, self.num_layers):55            z = zs[-layer]56            d = self.activation_derivative(z)57            delta = np.dot(self.weights[-layer + 1].T, delta) * d58 59            weight_gradients[-layer] = np.dot(delta, activations[-layer - 1].T)60            bias_gradients[-layer] = delta61 62        return (weight_gradients, bias_gradients)63 64    def adjust(self, lr, weight_gradients, bias_gradients):65        self.weights = [66            w - lr * nw for w, nw in67            zip(self.weights, weight_gradients)68        ]69 70        self.bias = [71            b - lr * nb for b, nb in72            zip(self.bias, bias_gradients)73        ]74