part3_neural_network_mnist_and_own_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# python notebook for Make Your Own Neural Network\n",
    "# code for a 3-layer neural network, and code for learning the MNIST dataset\n",
    "# this version trains using the MNIST dataset, then tests on our own images\n",
    "# (c) Tariq Rashid, 2016\n",
    "# license is GPLv2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy\n",
    "# scipy.special for the sigmoid function expit()\n",
    "import scipy.special\n",
    "# library for plotting arrays\n",
    "import matplotlib.pyplot\n",
    "# ensure the plots are inside this notebook, not an external window\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# helper to load data from PNG image files\n",
    "import scipy.misc\n",
    "# glob helps select multiple files using patterns\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# neural network class definition\n",
    "class neuralNetwork:\n",
    "    \n",
    "    \n",
    "    # initialise the neural network\n",
    "    def __init__(self, inputnodes, hiddennodes, outputnodes, learningrate):\n",
    "        # set number of nodes in each input, hidden, output layer\n",
    "        self.inodes = inputnodes\n",
    "        self.hnodes = hiddennodes\n",
    "        self.onodes = outputnodes\n",
    "        \n",
    "        # link weight matrices, wih and who\n",
    "        # weights inside the arrays are w_i_j, where link is from node i to node j in the next layer\n",
    "        # w11 w21\n",
    "        # w12 w22 etc \n",
    "        self.wih = numpy.random.normal(0.0, pow(self.inodes, -0.5), (self.hnodes, self.inodes))\n",
    "        self.who = numpy.random.normal(0.0, pow(self.hnodes, -0.5), (self.onodes, self.hnodes))\n",
    "\n",
    "        # learning rate\n",
    "        self.lr = learningrate\n",
    "        \n",
    "        # activation function is the sigmoid function\n",
    "        self.activation_function = lambda x: scipy.special.expit(x)\n",
    "        \n",
    "        pass\n",
    "\n",
    "    \n",
    "    # train the neural network\n",
    "    def train(self, inputs_list, targets_list):\n",
    "        # convert inputs list to 2d array\n",
    "        inputs = numpy.array(inputs_list, ndmin=2).T\n",
    "        targets = numpy.array(targets_list, ndmin=2).T\n",
    "        \n",
    "        # calculate signals into hidden layer\n",
    "        hidden_inputs = numpy.dot(self.wih, inputs)\n",
    "        # calculate the signals emerging from hidden layer\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        \n",
    "        # calculate signals into final output layer\n",
    "        final_inputs = numpy.dot(self.who, hidden_outputs)\n",
    "        # calculate the signals emerging from final output layer\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "        \n",
    "        # output layer error is the (target - actual)\n",
    "        output_errors = targets - final_outputs\n",
    "        # hidden layer error is the output_errors, split by weights, recombined at hidden nodes\n",
    "        hidden_errors = numpy.dot(self.who.T, output_errors) \n",
    "        \n",
    "        # update the weights for the links between the hidden and output layers\n",
    "        self.who += self.lr * numpy.dot((output_errors * final_outputs * (1.0 - final_outputs)), numpy.transpose(hidden_outputs))\n",
    "        \n",
    "        # update the weights for the links between the input and hidden layers\n",
    "        self.wih += self.lr * numpy.dot((hidden_errors * hidden_outputs * (1.0 - hidden_outputs)), numpy.transpose(inputs))\n",
    "        \n",
    "        pass\n",
    "\n",
    "    \n",
    "    # query the neural network\n",
    "    def query(self, inputs_list):\n",
    "        # convert inputs list to 2d array\n",
    "        inputs = numpy.array(inputs_list, ndmin=2).T\n",
    "        \n",
    "        # calculate signals into hidden layer\n",
    "        hidden_inputs = numpy.dot(self.wih, inputs)\n",
    "        # calculate the signals emerging from hidden layer\n",
    "        hidden_outputs = self.activation_function(hidden_inputs)\n",
    "        \n",
    "        # calculate signals into final output layer\n",
    "        final_inputs = numpy.dot(self.who, hidden_outputs)\n",
    "        # calculate the signals emerging from final output layer\n",
    "        final_outputs = self.activation_function(final_inputs)\n",
    "        \n",
    "        return final_outputs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# number of input, hidden and output nodes\n",
    "input_nodes = 784\n",
    "hidden_nodes = 200\n",
    "output_nodes = 10\n",
    "\n",
    "# learning rate\n",
    "learning_rate = 0.1\n",
    "\n",
    "# create instance of neural network\n",
    "n = neuralNetwork(input_nodes,hidden_nodes,output_nodes, learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load the mnist training data CSV file into a list\n",
    "training_data_file = open(\"mnist_dataset/mnist_train.csv\", 'r')\n",
    "training_data_list = training_data_file.readlines()\n",
    "training_data_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# train the neural network\n",
    "\n",
    "# epochs is the number of times the training data set is used for training\n",
    "epochs = 10\n",
    "\n",
    "for e in range(epochs):\n",
    "    # go through all records in the training data set\n",
    "    for record in training_data_list:\n",
    "        # split the record by the ',' commas\n",
    "        all_values = record.split(',')\n",
    "        # scale and shift the inputs\n",
    "        inputs = (numpy.asfarray(all_values[1:]) / 255.0 * 0.99) + 0.01\n",
    "        # create the target output values (all 0.01, except the desired label which is 0.99)\n",
    "        targets = numpy.zeros(output_nodes) + 0.01\n",
    "        # all_values[0] is the target label for this record\n",
    "        targets[int(all_values[0])] = 0.99\n",
    "        n.train(inputs, targets)\n",
    "        pass\n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading ...  my_own_images/2828_my_own_6.png\n",
      "0.01\n",
      "1.0\n",
      "loading ...  my_own_images/2828_my_own_5.png\n",
      "0.01\n",
      "0.868\n",
      "loading ...  my_own_images/2828_my_own_3.png\n",
      "0.01\n",
      "1.0\n",
      "loading ...  my_own_images/2828_my_own_4.png\n",
      "0.01\n",
      "0.930118\n",
      "loading ...  my_own_images/2828_my_own_2.png\n",
      "0.01\n",
      "1.0\n"
     ]
    }
   ],
   "source": [
    "# our own image test data set\n",
    "our_own_dataset = []\n",
    "\n",
    "# load the png image data as test data set\n",
    "for image_file_name in glob.glob('my_own_images/2828_my_own_?.png'):\n",
    "    \n",
    "    # use the filename to set the correct label\n",
    "    label = int(image_file_name[-5:-4])\n",
    "    \n",
    "    # load image data from png files into an array\n",
    "    print (\"loading ... \", image_file_name)\n",
    "    img_array = scipy.misc.imread(image_file_name, flatten=True)\n",
    "    \n",
    "    # reshape from 28x28 to list of 784 values, invert values\n",
    "    img_data  = 255.0 - img_array.reshape(784)\n",
    "    \n",
    "    # then scale data to range from 0.01 to 1.0\n",
    "    img_data = (img_data / 255.0 * 0.99) + 0.01\n",
    "    print(numpy.min(img_data))\n",
    "    print(numpy.max(img_data))\n",
    "    \n",
    "    # append label and image data  to test data set\n",
    "    record = numpy.append(label,img_data)\n",
    "    our_own_dataset.append(record)\n",
    "    \n",
    "    pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[  1.04770075e-02]\n",
      " [  3.10563984e-03]\n",
      " [  2.89956440e-03]\n",
      " [  6.71413536e-01]\n",
      " [  5.94502331e-03]\n",
      " [  3.01325959e-02]\n",
      " [  1.70101838e-03]\n",
      " [  1.11106700e-03]\n",
      " [  3.90449549e-04]\n",
      " [  1.34192975e-03]]\n",
      "network says  3\n",
      "match!\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAP8AAAD8CAYAAAC4nHJkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAADW9JREFUeJzt3X2IXOUVx/HfiRpijAlJM4bVqptIqIrYFMdQiZQ2tdWK\nGPuPNmjYgrgVa3wLUrHi2x8Sa1UKKYVU16YlNRZ8iyAtMRYkoUQ3kvqStMaGDU2ySSYoGhOlzfb0\nj73KqjvPTGbuzJ3d8/3AMDP33Lv3cNnf3jvzzM5j7i4A8UwougEAxSD8QFCEHwiK8ANBEX4gKMIP\nBEX4gaAIPxAU4QeCOradO5s5c6Z3d3e3c5dAKAMDAzpw4IDVs25T4TezSyT9StIxkh5z9+Wp9bu7\nu9Xf39/MLgEklMvlutdt+LLfzI6R9GtJP5B0tqTFZnZ2oz8PQHs185p/vqR33X2Hu/9H0hpJi/Jp\nC0CrNRP+UyT9e8TzXdmyzzGzXjPrN7P+SqXSxO4A5Knl7/a7+0p3L7t7uVQqtXp3AOrUTPh3Szp1\nxPOvZssAjAHNhP81SXPNbLaZTZT0I0lr82kLQKs1PNTn7kfM7EZJf9HwUF+fu7+dW2cAWqqpcX53\nf1HSizn1AqCN+HgvEBThB4Ii/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiK8ANB\ntfWru9GYoaGhZP3QoUNVa1OnTk1uu2bNmmR9y5Ytyfq0adOS9auuuqpqbc6cOclt0Vqc+YGgCD8Q\nFOEHgiL8QFCEHwiK8ANBEX4gKMb522Dr1q3J+urVq5P1DRs2JOu9vb1Va1dffXVy2zPOOCNZT32G\nQJIGBweT9bvuuqtq7aSTTkpuu3x5ctJnTZo0KVlHGmd+ICjCDwRF+IGgCD8QFOEHgiL8QFCEHwiq\nqXF+MxuQdFDSkKQj7l7Oo6nxZsKE9N/Yrq6uZP2JJ55I1ru7u4+2pc+cf/75TdWbsWDBgmT9scce\nS9ZvvPHGPNsJJ48P+XzH3Q/k8HMAtBGX/UBQzYbfJb1kZpvNrPpnTAF0nGYv+y90991mdpKkdWb2\nD3d/ZeQK2R+FXkk67bTTmtwdgLw0deZ3993Z/X5Jz0qaP8o6K9297O7lUqnUzO4A5Kjh8JvZCWZ2\n4qePJX1f0lt5NQagtZq57J8l6Vkz+/Tn/NHd/5xLVwBaruHwu/sOSV/PsZdx68wzz2yqPl5t3Lgx\nWd+5c2eyfvjw4WR98uTJR91TJAz1AUERfiAowg8ERfiBoAg/EBThB4Liq7vRsVasWJGsz507N1lP\nfaU5OPMDYRF+ICjCDwRF+IGgCD8QFOEHgiL8QFCM86Nj7dixI1m/+OKL29TJ+MSZHwiK8ANBEX4g\nKMIPBEX4gaAIPxAU4QeCYpwfhbnnnnuS9WOPTf96Lly4MM92wuHMDwRF+IGgCD8QFOEHgiL8QFCE\nHwiK8ANB1RznN7M+SZdJ2u/u52TLZkh6SlK3pAFJV7r7+61rE63y8ccfJ+vr1q1L1jds2JCsX3fd\ndVVrS5YsSW57+umnJ+sTJnDuakY9R+93ki75wrI7JK1397mS1mfPAYwhNcPv7q9Ieu8LixdJWpU9\nXiXpipz7AtBijV43zXL3wezxXkmzcuoHQJs0/aLJ3V2SV6ubWa+Z9ZtZf6VSaXZ3AHLSaPj3mVmX\nJGX3+6ut6O4r3b3s7uVSqdTg7gDkrdHwr5XUkz3ukfR8Pu0AaJea4TezJyX9TdLXzGyXmV0rabmk\n75nZdkkXZc8BjCE1x/ndfXGV0ndz7gUFWLFiRbJ+3333JeuHDh1K1qdMmVK1dvfddye3RWvxKQkg\nKMIPBEX4gaAIPxAU4QeCIvxAUHx1d3BLly5N1m+++eZkfc+ePcn6NddcU7V25MiR5Lb3339/so7m\ncOYHgiL8QFCEHwiK8ANBEX4gKMIPBEX4gaAY5w9u0qRJTW3f3d2drL/88stVa7W+9nvTpk3J+rnn\nnpusH3/88cl6dJz5gaAIPxAU4QeCIvxAUIQfCIrwA0ERfiAoxvnRUhMnTqxaW7hwYXLbCy64IFm/\n/vrrk/Wenp5kPTrO/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QVM1xfjPrk3SZpP3ufk627F5J10mq\nZKvd6e4vtqpJxPTQQw8l6xs3bmxTJ+NTPWf+30m6ZJTlj7r7vOxG8IExpmb43f0VSe+1oRcAbdTM\na/6lZvaGmfWZ2fTcOgLQFo2G/zeS5kiaJ2lQ0sPVVjSzXjPrN7P+SqVSbTUAbdZQ+N19n7sPufv/\nJP1W0vzEuivdvezu5VKp1GifAHLWUPjNrGvE0x9KeiufdgC0Sz1DfU9K+rakmWa2S9I9kr5tZvMk\nuaQBST9pYY8AWqBm+N198SiLH29BL8DnzJ9f9dWkJOnkk09uUyfjE5/wA4Ii/EBQhB8IivADQRF+\nICjCDwTFV3ejY23evDlZf+CBB5L1F154Ic92xh3O/EBQhB8IivADQRF+ICjCDwRF+IGgCD8QFOP8\nbbB3795kferUqcn65MmT82ynYwwNDSXrS5YsSdZrTdGNNM78QFCEHwiK8ANBEX4gKMIPBEX4gaAI\nPxAU4/xt8Oqrrybrt912W7L+zjvvJOsTJozNv+EHDx5M1h9+uOoscJKkRYsW5dlOOGPztwZA0wg/\nEBThB4Ii/EBQhB8IivADQRF+IKia4/xmdqqk30uaJcklrXT3X5nZDElPSeqWNCDpSnd/v3Wtjl2X\nX355sr59+/Zk/aKLLkrWn3vuuaq1Wt8VUMvhw4eT9W3btiXrN9xwQ9VaX19fclvG8VurnjP/EUnL\n3P1sSd+U9FMzO1vSHZLWu/tcSeuz5wDGiJrhd/dBd389e3xQ0jZJp0haJGlVttoqSVe0qkkA+Tuq\n1/xm1i3pG5I2SZrl7oNZaa+GXxYAGCPqDr+ZTZH0tKRb3P3DkTV3dw2/HzDadr1m1m9m/ZVKpalm\nAeSnrvCb2XEaDv5qd38mW7zPzLqyepek/aNt6+4r3b3s7uVSqZRHzwByUDP8ZmaSHpe0zd0fGVFa\nK6kne9wj6fn82wPQKvX8S+8CSUskvWlmW7Jld0paLulPZnatpJ2SrmxNi+PfsmXLkvVa/7K7Z8+e\nqrVPPvkkue3tt9+erO/bty9Znz59erL+4IMPVq2dddZZyW3RWjXD7+4bJFmV8nfzbQdAu/AJPyAo\nwg8ERfiBoAg/EBThB4Ii/EBQfHX3GHDrrbc2vO0HH3yQrN90003J+uzZs5P1GTNmHHVP6Ayc+YGg\nCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMb5x7lp06Yl6+edd16bOkGn4cwPBEX4gaAIPxAU4QeCIvxA\nUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8IivADQdUMv5mdamZ/NbOtZva2md2cLb/X\nzHab2Zbsdmnr2wWQl3q+zOOIpGXu/rqZnShps5mty2qPuvsvW9cegFapGX53H5Q0mD0+aGbbJJ3S\n6sYAtNZRveY3s25J35C0KVu01MzeMLM+M5teZZteM+s3s/5KpdJUswDyU3f4zWyKpKcl3eLuH0r6\njaQ5kuZp+Mrg4dG2c/eV7l5293KpVMqhZQB5qCv8ZnachoO/2t2fkSR33+fuQ+7+P0m/lTS/dW0C\nyFs97/abpMclbXP3R0Ys7xqx2g8lvZV/ewBapZ53+xdIWiLpTTPbki27U9JiM5snySUNSPpJSzoE\n0BL1vNu/QZKNUnox/3YAtAuf8AOCIvxAUIQfCIrwA0ERfiAowg8ERfiBoAg/EBThB4Ii/EBQhB8I\nivADQRF+ICjCDwRl7t6+nZlVJO0csWimpANta+DodGpvndqXRG+NyrO30929ru/La2v4v7Rzs353\nLxfWQEKn9tapfUn01qiieuOyHwiK8ANBFR3+lQXvP6VTe+vUviR6a1QhvRX6mh9AcYo+8wMoSCHh\nN7NLzOyfZvaumd1RRA/VmNmAmb2ZzTzcX3AvfWa238zeGrFshpmtM7Pt2f2o06QV1FtHzNycmFm6\n0GPXaTNet/2y38yOkfSOpO9J2iXpNUmL3X1rWxupwswGJJXdvfAxYTP7lqSPJP3e3c/Jlv1C0nvu\nvjz7wznd3X/WIb3dK+mjomduziaU6Ro5s7SkKyT9WAUeu0RfV6qA41bEmX++pHfdfYe7/0fSGkmL\nCuij47n7K5Le+8LiRZJWZY9XafiXp+2q9NYR3H3Q3V/PHh+U9OnM0oUeu0RfhSgi/KdI+veI57vU\nWVN+u6SXzGyzmfUW3cwoZmXTpkvSXkmzimxmFDVnbm6nL8ws3THHrpEZr/PGG35fdqG7z5P0A0k/\nzS5vO5IPv2brpOGaumZubpdRZpb+TJHHrtEZr/NWRPh3Szp1xPOvZss6grvvzu73S3pWnTf78L5P\nJ0nN7vcX3M9nOmnm5tFmllYHHLtOmvG6iPC/Jmmumc02s4mSfiRpbQF9fImZnZC9ESMzO0HS99V5\nsw+vldSTPe6R9HyBvXxOp8zcXG1maRV87Dpuxmt3b/tN0qUafsf/X5J+XkQPVfqaI+nv2e3tonuT\n9KSGLwP/q+H3Rq6V9BVJ6yVtl/SSpBkd1NsfJL0p6Q0NB62roN4u1PAl/RuStmS3S4s+dom+Cjlu\nfMIPCIo3/ICgCD8QFOEHgiL8QFCEHwiK8ANBEX4gKMIPBPV/OtAMJKwFHisAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fd6554d1940>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# test the neural network withour own images\n",
    "\n",
    "# record to test\n",
    "item = 2\n",
    "\n",
    "# plot image\n",
    "matplotlib.pyplot.imshow(our_own_dataset[item][1:].reshape(28,28), cmap='Greys', interpolation='None')\n",
    "\n",
    "# correct answer is first value\n",
    "correct_label = our_own_dataset[item][0]\n",
    "# data is remaining values\n",
    "inputs = our_own_dataset[item][1:]\n",
    "\n",
    "# query the network\n",
    "outputs = n.query(inputs)\n",
    "print (outputs)\n",
    "\n",
    "# the index of the highest value corresponds to the label\n",
    "label = numpy.argmax(outputs)\n",
    "print(\"network says \", label)\n",
    "# append correct or incorrect to list\n",
    "if (label == correct_label):\n",
    "    print (\"match!\")\n",
    "else:\n",
    "    print (\"no match!\")\n",
    "    pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}