Backpropagation from scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "y871RZtRhCEH"
   },
   "source": [
    "# <font color='red'>Backpropagation from Scratch</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kyCXwHIt_Hmj",
    "outputId": "dae68ede-7334-4f02-d9cf-2251c36b8f1f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mounted at /content/drive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0huUQ0byiI0I"
   },
   "source": [
    "## <font color='red'>Loading data </font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "1hSje5CBgcUb",
    "outputId": "bbe635ed-1c03-46c3-be61-e01309d3a350"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(506, 6)\n",
      "(506, 5) (506,)\n"
     ]
    }
   ],
   "source": [
    "import pickle\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "with open('/content/drive/MyDrive/aaic/data.pkl', 'rb') as f:\n",
    "    data = pickle.load(f)\n",
    "print(data.shape)\n",
    "X = data[:, :5]\n",
    "y = data[:, -1]\n",
    "print(X.shape, y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "5JL-0soQistC"
   },
   "source": [
    "# <font color='red'>Computational graph</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nREnTTJ3i0Vd"
   },
   "source": [
    "<img src='https://i.imgur.com/seSGbNS.png'>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "DSPX_H_4i_HT"
   },
   "source": [
    "\n",
    "*  **If you observe the graph, we are having input features [f1, f2, f3, f4, f5] and 9 weights [w1, w2, w3, w4, w5, w6,    w7, w8, w9]**.<br><br>\n",
    "*  **The final output of this graph is a value L which is computed as (Y-Y')^2** "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "D54eDEv6jkO4"
   },
   "source": [
    "## <font color='red'>Task 1: Implementing Forward propagation, Backpropagation and Gradient checking </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "xyWK94o_fTKH"
   },
   "source": [
    "# <font color='red'>Task 1.1 </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RwEcPWLffTKI"
   },
   "source": [
    "## <font color='blue'>Forward propagation </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FCZrm-gkfTKI"
   },
   "source": [
    "\n",
    "*  <b>\n",
    "    Forward propagation</b>\n",
    "    For easy debugging, we will break the computational graph into 3 parts.\n",
    "\n",
    "    <font color='green'><b>Part 1</b></font></b>\n",
    "    <img src='https://i.imgur.com/0xUaxy6.png'><br><br>\n",
    "    <font color='green'><b>Part 2</b></font></b><br>\n",
    "    <img src='https://i.imgur.com/J29pAJL.png'><br><br>\n",
    "    <font color='green'><b>Part 3</b></font></b>\n",
    "    <img src='https://i.imgur.com/vMyCsd9.png'>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "ADCovl2FfTKJ"
   },
   "outputs": [],
   "source": [
    "def sigmoid(z):\n",
    "    '''In this function, we will compute the sigmoid(z)'''\n",
    "    return 1/(1+np.exp(-z))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "id": "KD3piNkifTKJ"
   },
   "outputs": [],
   "source": [
    "def forward_propagation(x, y, w):\n",
    "        '''In this function, we will compute the forward propagation '''\n",
    "        # X: input data point, note that in this assignment you are having 5-d data points\n",
    "        # y: output varible\n",
    "        # W: weight array, its of length 9, W[0] corresponds to w1 in graph, W[1] corresponds to w2 in graph,..., W[8] corresponds to w9 in graph.  \n",
    "        # you have to return the following variables\n",
    "        # exp= part1 (compute the forward propagation until exp and then store the values in exp)\n",
    "        # tanh =part2(compute the forward propagation until tanh and then store the values in tanh)\n",
    "        # sig = part3(compute the forward propagation until sigmoid and then store the values in sig)\n",
    "        # we are computing one of the values for better understanding\n",
    "        \n",
    "        val_1= (w[0]*x[0]+w[1]*x[1]) * (w[0]*x[0]+w[1]*x[1]) + w[5]\n",
    "        part_1 = np.exp(val_1)\n",
    "        part_2 = np.tanh(part_1+w[6])\n",
    "        part_3 = sigmoid(np.sin(w[2]*x[2])*(w[3]*x[3]+w[4]*x[4])+w[7])\n",
    "\n",
    "        total=part_3*w[8]+part_2\n",
    "\n",
    "        L=(y-total)**2\n",
    "\n",
    "        dy_pred=-2*(y-total)\n",
    "        \n",
    "        # after computing part1,part2 and part3 compute the value of y' from the main Computational graph using required equations\n",
    "        # write code to compute the value of L=(y-y')^2 and store it in variable loss\n",
    "        # compute derivative of L  w.r.to y' and store it in dy_pred \n",
    "        # Create a dictionary to store all the intermediate values i.e. dy_pred ,loss,exp,tanh,sigmoid\n",
    "        # we will be using the dictionary to find values in backpropagation, you can add other keys in dictionary as well\n",
    "        \n",
    "        forward_dict={}\n",
    "        forward_dict['exp']= part_1\n",
    "        forward_dict['sigmoid'] =part_3\n",
    "        forward_dict['tanh'] =part_2\n",
    "        forward_dict['loss'] =L\n",
    "        forward_dict['dy_pred'] =dy_pred \n",
    "        #print(forward_dict)\n",
    "        return forward_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "2zcQmvkvfTKL"
   },
   "source": [
    "# <font color='red'>Task 1.2 </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "4YVYD18xfTKL"
   },
   "source": [
    "## <font color='blue'>Backward propagation</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "Ld1PMQtffTKL"
   },
   "outputs": [],
   "source": [
    "def backward_propagation(x,y,w,forward_dict):\n",
    "    '''In this function, we will compute the backward propagation '''\n",
    "    # forward_dict: the outputs of the forward_propagation() function\n",
    "    # write code to compute the gradients of each weight [w1,w2,w3,...,w9]\n",
    "    # Hint: you can use dict type to store the required variables \n",
    "    # dw1 = # in dw1 compute derivative of L w.r.to w1\n",
    "    # dw2 = # in dw2 compute derivative of L w.r.to w2\n",
    "    # dw3 = # in dw3 compute derivative of L w.r.to w3\n",
    "    # dw4 = # in dw4 compute derivative of L w.r.to w4\n",
    "    # dw5 = # in dw5 compute derivative of L w.r.to w5\n",
    "    # dw6 = # in dw6 compute derivative of L w.r.to w6\n",
    "    # dw7 = # in dw7 compute derivative of L w.r.to w7\n",
    "    # dw8 = # in dw8 compute derivative of L w.r.to w8\n",
    "    # dw9 = # in dw9 compute derivative of L w.r.to w9\n",
    "    \n",
    "    backward_dict={}\n",
    "    #store the variables dw1,dw2 etc. in a dict as backward_dict['dw1']= dw1,backward_dict['dw2']= dw2...\n",
    "    #dL/dw9 = dL/dy' * dy'/dw9\n",
    "    #dL/dw8 = dL/dy' * dy'/dw8 \n",
    "    #dL/dw7 = dL/dy' * dy'/dw7\n",
    "    #dl/dw6 = dL/dy' * dy'/dw6\n",
    "    backward_dict['dw1'] = forward_dict['dy_pred'] * (1-forward_dict['tanh']**2) * forward_dict['exp'] * 2 * (w[0]*x[0]+w[1]*x[1]) * x[0]\n",
    "    backward_dict['dw2'] = forward_dict['dy_pred'] * (1-forward_dict['tanh']**2) * forward_dict['exp'] * 2 * (w[0]*x[0]+w[1]*x[1]) * x[1]\n",
    "    backward_dict['dw3'] = forward_dict['dy_pred'] * w[8] * forward_dict['sigmoid']*(1-forward_dict['sigmoid'])*(w[3]*x[3]+w[4]*x[4])*np.cos(w[2]*x[2])*x[2]\n",
    "    backward_dict['dw4'] = forward_dict['dy_pred'] * w[8] * forward_dict['sigmoid']*(1-forward_dict['sigmoid'])*np.sin(w[2]*x[2])*x[3]\n",
    "    backward_dict['dw5'] = forward_dict['dy_pred'] * w[8] * forward_dict['sigmoid']*(1-forward_dict['sigmoid'])*np.sin(w[2]*x[2])*x[4]\n",
    "    backward_dict['dw6'] = forward_dict['dy_pred'] * (1-forward_dict['tanh']**2)* forward_dict['exp']\n",
    "    backward_dict['dw7'] = forward_dict['dy_pred'] * (1-forward_dict['tanh']**2)\n",
    "    backward_dict['dw8'] = forward_dict['dy_pred'] * w[8] * forward_dict['sigmoid']*(1-forward_dict['sigmoid'])\n",
    "    backward_dict['dw9'] = forward_dict['dy_pred'] * forward_dict['sigmoid']\n",
    "    return backward_dict"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RlR1JVN5fTKM"
   },
   "source": [
    "# <font color='red'>Task 1.3 </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "STS4NrQQc6OH"
   },
   "source": [
    " ## <font color='blue'>Gradient clipping</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "XY7ivRNzdPok"
   },
   "source": [
    "<b> Check this  <a href='https://towardsdatascience.com/how-to-debug-a-neural-network-with-gradient-checking-41deec0357a9'>blog link</a> for more details on Gradient clipping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "BrsfpDoidtZ5"
   },
   "source": [
    " we know that the derivative of any function is\n",
    " \n",
    " $$\\lim_{\\epsilon\\to0}\\frac{f(x+\\epsilon)-f(x-\\epsilon)}{2\\epsilon}$$\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "vUcmt0kPd02f"
   },
   "source": [
    "*  The definition above can be used as a numerical approximation of the derivative. Taking an epsilon small enough, the calculated approximation will have an error in the range of epsilon squared. \n",
    "\n",
    "*  In other words, if epsilon is 0.001, the approximation will be off by 0.00001.\n",
    "\n",
    "Therefore, we can use this to approximate the gradient, and in turn make sure that backpropagation is implemented properly. This forms the basis of <b>gradient checking!</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "pFSu16KCeU0x"
   },
   "source": [
    "## <Font color='blue'>Gradient checking example</font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Gz0mmT_xecfC"
   },
   "source": [
    "<font >\n",
    "lets understand the concept with a simple example:\n",
    "$f(w1,w2,x1,x2)=w_{1}^{2} . x_{1} + w_{2} . x_{2}$ \n",
    "\n",
    "from the above function , lets assume $w_{1}=1$, $w_{2}=2$, $x_{1}=3$, $x_{2}=4$ the gradient of $f$ w.r.t $w_{1}$ is\n",
    "\n",
    "\\begin{array} {lcl}\n",
    "\\frac{df}{dw_{1}} = dw_{1} &=&2.w_{1}.x_{1} \\\\& = &2.1.3\\\\& = &6\n",
    "\\end{array}\n",
    "\n",
    "\n",
    "let calculate the aproximate gradient of $w_{1}$ as mentinoned in the above formula and considering $\\epsilon=0.0001$\n",
    "\n",
    "\\begin{array} {lcl}\n",
    "dw_1^{approx} & = & \\frac{f(w1+\\epsilon,w2,x1,x2)-f(w1-\\epsilon,w2,x1,x2)}{2\\epsilon} \\\\ & = & \\frac{((1+0.0001)^{2} . 3 + 2 . 4) - ((1-0.0001)^{2} . 3 + 2 . 4)}{2\\epsilon} \\\\ & = & \\frac{(1.00020001 . 3 + 2 . 4) - (0.99980001. 3 + 2 . 4)}{2*0.0001} \\\\ & = & \\frac{(11.00060003) - (10.99940003)}{0.0002}\\\\ & = & 5.99999999999\n",
    "\\end{array}\n",
    "\n",
    "Then, we apply the following formula for gradient check: <i>gradient_check</i> = \n",
    "$\\frac{\\left\\Vert\\left (dW-dW^{approx}\\rm\\right) \\right\\Vert_2}{\\left\\Vert\\left (dW\\rm\\right) \\right\\Vert_2+\\left\\Vert\\left (dW^{approx}\\rm\\right) \\right\\Vert_2}$\n",
    "\n",
    "The equation above is basically the Euclidean distance normalized by the sum of the norm of the vectors. We use normalization in case that one of the vectors is very small.\n",
    "As a value for epsilon, we usually opt for 1e-7. Therefore, if gradient check return a value less than 1e-7, then it means that backpropagation was implemented correctly. Otherwise, there is potentially a mistake in your implementation. If the value exceeds 1e-3, then you are sure that the code is not correct.\n",
    "\n",
    "in our example: <i>gradient_check</i> $ = \\frac{(6 - 5.999999999994898)}{(6 + 5.999999999994898)} = 4.2514140356330737e^{-13}$\n",
    "\n",
    "you can mathamatically derive the same thing like this\n",
    "\n",
    "\\begin{array} {lcl}\n",
    "dw_1^{approx} & = & \\frac{f(w1+\\epsilon,w2,x1,x2)-f(w1-\\epsilon,w2,x1,x2)}{2\\epsilon} \\\\ & = & \\frac{((w_{1}+\\epsilon)^{2} . x_{1} + w_{2} . x_{2}) - ((w_{1}-\\epsilon)^{2} . x_{1} + w_{2} . x_{2})}{2\\epsilon} \\\\ & = & \\frac{4. \\epsilon.w_{1}. x_{1}}{2\\epsilon} \\\\ & = &  2.w_{1}.x_{1}\n",
    "\\end{array}\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "f1BFQQaCermK"
   },
   "source": [
    "## <font color='red'> Implement Gradient checking </font> <br>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "GqpfA3AqfJba"
   },
   "source": [
    "**Algorithm** \n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "FL39KeRFfNoD"
   },
   "source": [
    "<pre>\n",
    "<font color='darkblue'>\n",
    "W = initilize_randomly\n",
    "def gradient_checking(data_point, W):<font color='grey'>\n",
    "    # compute the L value using forward_propagation()\n",
    "    # compute the gradients of W using backword_propagation()</font>\n",
    "    approx_gradients = []\n",
    "    for each wi weight value in W:<font color='grey'>\n",
    "        # add a small value to weight wi, and then find the values of L with the updated weights\n",
    "        # subtract a small value to weight wi, and then find the values of L with the updated weights\n",
    "        # compute the approximation gradients of weight wi</font>\n",
    "        approx_gradients.append(approximation gradients of weight wi)<font color='grey'>\n",
    "    # compare the gradient of weights W from backword_propagation() with the aproximation gradients of weights with <br>  gradient_check formula</font>\n",
    "    return gradient_check</font>\n",
    "<b>NOTE: you can do sanity check by checking all the return values of gradient_checking(),<br> they have to be zero. if not you have bug in your code\n",
    "</pre></b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "XBoJRqAwfTKO"
   },
   "outputs": [],
   "source": [
    "def gradient_checking(x,y,w,eps):\n",
    "    # compute the dict value using forward_propagation()\n",
    "    # compute the actual gradients of W using backword_propagation()\n",
    "    forward_dict=forward_propagation(x,y,w)\n",
    "    backward_dict=backward_propagation(x,y,w,forward_dict)\n",
    "    \n",
    "    #we are storing the original gradients for the given datapoints in a list\n",
    "    original_gradients_list=list(backward_dict.values())\n",
    "    # make sure that the order is correct i.e. first element in the list corresponds to  dw1 ,second element is dw2 etc.\n",
    "    # you can use reverse function if the values are in reverse order\n",
    "    \n",
    "    approx_gradients_list=[]\n",
    "    #now we have to write code for approx gradients, here you have to make sure that you update only one weight at a time\n",
    "    #write your code here and append the approximate gradient value for each weight in  approx_gradients_list\n",
    "    for i in range(len(w)):\n",
    "        temp=w.copy()\n",
    "        temp[i]+=eps\n",
    "        f=forward_propagation(x,y,temp)\n",
    "        f=f['loss']\n",
    "        temp=w.copy()\n",
    "        temp[i]-=eps\n",
    "        f1=forward_propagation(x,y,temp)\n",
    "        f1=f1['loss']\n",
    "        approx_gradients_list.append((f-f1)/(2*eps))\n",
    "    #performing gradient check operation\n",
    "    original_gradients_list=np.array(original_gradients_list)\n",
    "    approx_gradients_list=np.array(approx_gradients_list)\n",
    "    gradient_check_value =(original_gradients_list-approx_gradients_list)/(original_gradients_list+approx_gradients_list)\n",
    "    return gradient_check_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "wCmoPG89dgEH"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "0ufbFBFfhgL0"
   },
   "source": [
    "# <font color='red'> Task 2 : Optimizers </font>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "7zt05soYh1RM"
   },
   "source": [
    "* As a part of this task, we will be implementing 2  optimizers(methods to update weight)\n",
    "* Use the same computational graph that was mentioned above to do this task\n",
    "* The weights have been initialized from normal distribution with mean=0 and std=0.01. The initialization of weights is very important otherwiswe we can face vanishing gradient and exploding gradients problem."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jmefh7ktjbaR"
   },
   "source": [
    "<font color='blue'><b>Algorithm</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "nAc4NudkjdNa"
   },
   "source": [
    "<pre>\n",
    "    for each epoch(1-20):\n",
    "        for each data point in your data:\n",
    "            using the functions forward_propagation() and backword_propagation() compute the gradients of weights\n",
    "            update the weigts with help of gradients  \n",
    "</pre>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "_fdmPNqtjm3X"
   },
   "source": [
    "## <font color='red'> Implement below tasks</b>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "Ll9-CRsLjx_D"
   },
   "source": [
    "\n",
    "*  <b>Task 2.1</b>: we will be implementing the above algorithm with <b>Vanilla update</b> of weights<br><br>\n",
    "*  <b>Task 2.2</b>: we will be implementing the above algorithm with <b>Momentum update</b> of weights<br><br>\n",
    "*  <b>Task 2.3</b>: we will be implementing the above algorithm with <b>Adam update</b> of weights<br><br>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "cv_hZFWalS2z"
   },
   "source": [
    "### <font color='blue'>2.1 Algorithm with Vanilla update of weights</font>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "id": "puDEWK7CSn6k"
   },
   "outputs": [],
   "source": [
    "def vanilla(x,y,epochs=100,learningrate=0.01):\n",
    "  loss=[]\n",
    "  learningrate=0.01\n",
    "  w=np.random.normal(0,0.01,9)\n",
    "  for i in range(epochs):\n",
    "    a=forward_propagation(x[0],y[0],w)\n",
    "    for i in a:\n",
    "      a[i]=0\n",
    "    for i in range(len(x)):\n",
    "      fp=forward_propagation(x[i],y[i],w)\n",
    "      for i in fp:\n",
    "        a[i]=fp[i]+a[i]\n",
    "    for i in a:\n",
    "      a[i]=a[i]/len(x)\n",
    "    loss.append(a['loss'])\n",
    "    b=backward_propagation(x[0],y[0],w,a)\n",
    "    for i in b:\n",
    "      b[i]=0\n",
    "    for i in range(len(x)):\n",
    "      bp=backward_propagation(x[i],y[i],w,a)\n",
    "      for i in bp:\n",
    "        b[i]=b[i]+bp[i]\n",
    "    for i in b:\n",
    "      b[i]=b[i]/len(x)\n",
    "    new_weights=[]\n",
    "    for vv in range(len(b)):\n",
    "      new_weights.append(b[f'dw{vv+1}'])\n",
    "    new_weights=np.array(new_weights)\n",
    "    w=w-learningrate*new_weights\n",
    "  return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "id": "PI9aqEzpQv9j"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "id": "q8sLTig2Pw-9"
   },
   "outputs": [],
   "source": [
    "loss=vanilla(X,y,100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 279
    },
    "id": "SSKIcDeIQ10U",
    "outputId": "02c83e4c-96bb-43cf-d412-449941c78464"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAXdUlEQVR4nO3df5Sc1X3f8fd3VwgbkM0ibIUiLPFDgRhagyRjuaTpEsccTGlIbae2ITRxofwDp47tOrWPfZyU9g+nTZM6jQ6JCsTGpSgu2LGOSo0LZe3TnApLIjYF2Y6FjLAIFAcvBJnYq9V++8c8646X3dXs7jw7M899v87Zo52ZZ2buPY+kz7n3e5/7RGYiSSrXUK8bIEnqLYNAkgpnEEhS4QwCSSqcQSBJhVvR6wYs1Kmnnprr169f1Ht/8IMfcOKJJ3a3QQOgxH6X2Gcos98l9hkW3u+9e/f+VWa+ZrbXBi4I1q9fz549exb13rGxMUZHR7vboAFQYr9L7DOU2e8S+wwL73dEHJzrNaeGJKlwBoEkFc4gkKTCGQSSVDiDQJIKZxBIUuGKCYK9B8fZ+fgEew+O97opktRXigiCvQfHuebWXdzz7SNcc+suw0CS2hQRBLsOPMfE5BQJHJmcYteB53rdJEnqG0UEwZazVrNyxRBDwHErhthy1upeN0mS+kYRQbBp3Qh3Xr+Ft284jjuv38KmdSO9bpIk9Y2B22tosTatG+HFs1caApI0QxEjgpn2Hhxn64P7LRpLEgWNCKZNryCamJxi5Yohp4okFa+4EcH0CqKpdAWRJEGBQTC9gmg4XEEkSVDg1ND0CqJdB55jy1mrnRaSVLziggBaYWAASFJLcVNDM7mCSFLpihwRTHMFkSQVPiJwBZEkFR4EriCSpMKnhlxBJEmFBwG4gkiSapsaiojbI+LZiHh0jtcjIn4/IvZHxCMRsbGutiyEq4gklabOEcGngD8A7pjj9bcBG6qfNwG3VH/2jKuIJJWothFBZn4F+P48h1wF3JEtu4CTI+K0utrTCVcRSSpRL2sEpwPfbXt8qHru6ZkHRsQNwA0Aa9asYWxsbFFfePjw4Xnfe/zzR1kRMJkwHHD88wcZGzu0qO/qJ8fqdxOV2Gcos98l9hm62++BKBZn5jZgG8DmzZtzdHR0UZ8zNjbGfO8dBS7aON64VUTH6ncTldhnKLPfJfYZutvvXgbBU8AZbY/XVs/1VPsqor0HmxcKkjRTL4NgB3BTRGynVSR+ITNfNi3UKxaOJZWitiCIiLtozbacGhGHgN8EjgPIzD8E7gWuAPYDLwHvrastizFb4dggkNREtQVBZr7nGK8ncGNd379U09tPHJmccvsJSY02EMXiXnD7CUmlMAjmYeFYUgkMgg5YOJbUZEVvQ90prziW1GQGQQe8b4GkJnNqqAMWjiU1mUHQIQvHkprKIFggC8eSmsYawQJZOJbUNAbBAlk4ltQ0Tg0tkIVjSU1jECyChWNJTWIQLIGFY0lNYI1gCSwcS2oCg2AJLBxLagKnhpZgZuEYYOuD+60XSBooBsESTReOrRdIGlRODXWJ9QJJg8og6BLrBZIGlVNDXeKFZpIGlUHQRV5oJmkQGQQ1sHAsaZBYI6iBhWNJg8QgqIGFY0mDxKmhGnihmaRBYhDUxAvNJA0Kp4ZqZr1AUr8zCGpmvUBSv3NqqGbWCyT1O4NgGVgvkNTPnBpaRtYLJPUjg2AZzawXjJywkq0P7mfvwfFeN01SwZwaWkbt9YKRE1Zy887HnCaS1HOOCJbZpnUj3HjpOYy/NOE0kaS+YBD0iMtKJfULp4Z6xGWlkvqFQdBDLiuV1A+cGuoDLiuV1EsGQR+wXiCpl2oNgoi4PCK+FRH7I+LDs7z+uoh4MCL+PCIeiYgr6mxPv5quF3zgsnO58/otAF5fIGnZ1FYjiIhhYCvwVuAQsDsidmTmvrbDPgZ8NjNviYjXA/cC6+tqUz+zXiCpV+ocEVwM7M/MA5k5AWwHrppxTAKvqn5/NfCXNbZnIFgvkLTcIjPr+eCIdwKXZ+b11eNrgTdl5k1tx5wGfAkYAU4EfiEz987yWTcANwCsWbNm0/bt2xfVpsOHD3PSSSct6r3LZf/4Uf7t7h8yOQUrhuDq81Zy+Ehy3inDnDMyvKjPHIR+d1uJfYYy+11in2Hh/b700kv3Zubm2V7r9fLR9wCfysx/HxFvBj4TERdk5lT7QZm5DdgGsHnz5hwdHV3Ul42NjbHY9y6XUeCijeOzbENxdNHTRIPQ724rsc9QZr9L7DN0t991Tg09BZzR9nht9Vy764DPAmTm/wZeAZxaY5sGgttQSFpOdQbBbmBDRJwZESuBdwM7ZhzzJPAWgIj4GVpB8L0a2zRQ3K1U0nKobWooMycj4ibgPmAYuD0zH4uIm4E9mbkD+CDwnyLi/bQKx7+WdRUtBpC7lUpaDrXWCDLzXlpLQtuf+3jb7/uAS+psw6CbXla69cH9L5smMggkdYNXFg8Irz6WVJderxpSh2buVjp98Vn7Y0laDINggExPEwFegSypa5waGlBegSypWwyCAeXSUknd4tTQgHJpqaRucUQwwLwCWVI3GAQN4DSRpKVwaqgBjjVNJEnzcUTQEE4TSVosg6BhZpsm2vn4hNNEkubk1FDDzDZN9KMjU+x8YperiSTNyhFBA82cJkqcJpI0N4OgwaaniYZwNZGkuTk11GDT00R33b+bjRec50VnkmZlEDTcpnUjvHj2Sh6bsZronocPuXOpJMAgKMb0NNGRySmGh4K79x5i8qijA0nWCIoxPU30gcvO5Zc3n8HkUa81kNRiEBRkejXR2zeudUsKST/m1FCB3LlUUruORgQR8b6IeFW03BYRD0fEZXU3TvVxSwpJ0zqdGvqnmfnXwGXACHAt8InaWqVl486lkjqdGorqzyuAz2TmYxER871Bg8FpIkmdjgj2RsSXaAXBfRGxCpiqr1laTnNNE93z8CFHB1IBOh0RXAdcCBzIzJci4hTgvfU1S73gtQZSmTodEbwZ+FZmPh8RvwJ8DHihvmapF7zWQCpTp0FwC/BSRLwB+CDwOHBHba1Sz3itgVSeTqeGJjMzI+Iq4A8y87aIuK7Ohqm3LCJL5eh0RPBiRHyE1rLR/xYRQ8Bx9TVL/cAislSGTkcE7wKupnU9wTMR8Trg39XXLPUTi8hSs3U0IsjMZ4A7gVdHxJXADzPTGkEh5isiOzqQBl9HI4KI+Me0RgBjtC4u+48R8aHMvLvGtqmPbFo3wqZ1I+w9OM49Dx9ydCA1SKdTQx8F3piZzwJExGuA+wGDoDDtReS/fP5vuOurT/7EElODQBo8nRaLh6ZDoPLcAt6rhnGJqdQsnY4IvhgR9wF3VY/fBdxbT5M0KFxiKjVDR0GQmR+KiHcAl1RPbcvMz9fXLA2K6drB1gf3z7qdtfdFlvpfxzemycx7gHtqbIsGWPsS0+lpomtu3eUIQRoA8wZBRLwI5GwvAZmZr6qlVRo47dNEW85aza4Dz73sIjRHB1J/mjcIMnPVUj48Ii4HPgkMA7dm5stuZlMtTf0tWoHz9cy8einfqd6Zniaa5kVo0mCo7Z7FETEMbAXeChwCdkfEjszc13bMBuAjwCWZOR4Rr62rPVpe8y0zdXQg9Zc6b15/MbA/Mw8ARMR24CpgX9sx/wzYmpnjADOWqGrAeRGaNBjqvBbgdOC7bY8PVc+1+2ngpyPizyJiVzWVpIZxiwqpv9U5Iuj0+zcAo8Ba4CsR8bcz8/n2gyLiBuAGgDVr1jA2NraoLzt8+PCi3zvI+qXf5wccz1FWBExma8XBZ3c/ydEpOG4IfuONr+CckeGufFe/9Hm5ldjvEvsM3e13nUHwFHBG2+O11XPtDgEPZeYR4DsR8Re0gmF3+0GZuQ3YBrB58+YcHR1dVIPGxsZY7HsHWT/1exS4aOP4T9QOEjiacIDX8qN8ZVdqB/3U5+VUYr9L7DN0t991BsFuYENEnEkrAN5Nayvrdn8KvAf444g4ldZU0YEa26Q+YO1A6i+1BUFmTkbETcB9tJaP3p6Zj0XEzcCezNxRvXZZROwDjgIfykxvjlsIVxZJ/aHWGkFm3suMPYky8+NtvyfwgepHBXJ0IPVer4vFEuDoQOolg0B9w9GB1BsGgfqOowNpeRkE6kuODqTlYxCorzk6kOpnEKjvOTqQ6mUQaGA4OpDqYRBooHQ6OpDUOYNAA+lYo4OJ70+w6sxxRwdSB+rchlqq1aZ1I9x46Tm8feNaVq4YYjj48ejgnm8f4Zpbd7m9tdQBRwQaeLONDhJrB1KnDAI1wszawcSRl9cOPn7l+Yy/NGEoSDMYBGqU6dHBXffv5vhTTvtx7WDiyBQf/8KjTGW63FSawRqBGmfTuhGuPHvlT9QOhoaCqUxvkSnNwhGBGqu9djBywkpu3vmYF6NJszAI1GjTtQOAc39q1azLTXcdaN0LyaKySmUQqBizXYx23IohRk5YyTW37mJi0qKyymQQqDjtU0ZbzlrNrgPPMTE5ZVFZxTIIVKT2KSOAlSuGODI5RcTLi8pOGanpDAIVz6KySmcQSHRWVHZ0oKYyCKQZOtnh1IKymsQgkOYw1w6nFpTVNAaBNI/ZRgcWlNU0BoHUgU4Lyk4ZaRAZBFKHjlVQdspIg8ogkBbBKSM1iUEgLYFTRmoCg0BaIqeMNOgMAqmLnDLSIDIIpBo4ZaRBYhBINVnolJGhoF4xCKRlcKwpI+sI6iWDQFpGc00ZWUdQLxkE0jKbbcroWNtfg7fSVH0MAqmHOt3++nMPH/JWmqqNQSD1ibnuqRzgrTRVK4NA6jMz76kMeE2CamUQSH1o5j2VO70m4eHHJ1h15rihoAWpNQgi4nLgk8AwcGtmfmKO494B3A28MTP31NkmaRB1ek3C0alk5xO7rCNoQWoLgogYBrYCbwUOAbsjYkdm7ptx3CrgfcBDdbVFapL5rklIvFBNC1fniOBiYH9mHgCIiO3AVcC+Gcf9a+C3gQ/V2BapcWa7JmHiyBRDQ16opoWJzKzngyPeCVyemddXj68F3pSZN7UdsxH4aGa+IyLGgH8x29RQRNwA3ACwZs2aTdu3b19Umw4fPsxJJ520qPcOshL7XWKf948f5evP/A2rTzye//LNCSanIAKmEhIYAv7+2mFWv3KI804Z5pyR4V43uStKPNew8H5feumlezNz82yv9axYHBFDwO8Cv3asYzNzG7ANYPPmzTk6Orqo7xwbG2Ox7x1kJfa7xD6P8v/7/Q8Pjs9aXP6zZ5LJo0dYueJoY6aMSjzX0N1+1xkETwFntD1eWz03bRVwATAWEQA/BeyIiF+0YCwtjRveaSHqDILdwIaIOJNWALwbuHr6xcx8ATh1+vF8U0OSFm+hG94ZCuWpLQgyczIibgLuo7V89PbMfCwibgb2ZOaOur5b0st1suGdoVCmWmsEmXkvcO+M5z4+x7GjdbZF0rE3vHNr7DJ5ZbFUqIWEgttZNJtBIGnBW2M7ZdQsBoGkn+CKo/IYBJLmtJhbbII30Rk0BoGkY1rILTa9ic7gMQgkdeRYdYRj3UTHUOhfBoGkBZstFOa7iY6h0N8MAklL0slNdAyF/mYQSOqqpVy0Zij0hkEgqTaGwmAwCCQtC0OhfxkEkpadodBfDAJJPbXUUHj48QlWnTluKCyBQSCpbywmFI5OJTuf2OVIYQkMAkl9qdNQSJw+WiqDQFLfmy8UJo5MMTRkTWEpDAJJA2VmKNx1/242XnBexzfXATfFm8kgkDSwNq0b4cWzVzL6ptd1fHOduTbFg3IDwiCQ1AhL2RRvxVBARLE33jEIJDXOQjfFO3I0gXILzwaBpEbrZFO84WpEcPRomRezGQSSijLfaKHUK5wNAknFmjlaKHXbC4NAkmbo1l5IMBgrkQwCSZrHYkNhkFYiGQSS1KGFhMIgrUQyCCRpEY4VCotdibT34PiyTycZBJK0RN1ciXTzzsdmvfK5zlAwCCSpixa7EunI5BT//dGnZ73yue5QMAgkaRl0sgXG2y44jd1PfL/jzfO6xSCQpGU211TSpnUjHY0cdh14jvOje+0xCCSph2abSjrWyGHLWat58TuHutYGg0CS+tR8I4ex73TvewwCSRoAM0cO3TRUy6dKkgaGQSBJhTMIJKlwBoEkFc4gkKTCGQSSVLjIzF63YUEi4nvAwUW+/VTgr7rYnEFRYr9L7DOU2e8S+wwL7/e6zHzNbC8MXBAsRUTsyczNvW7Hciux3yX2Gcrsd4l9hu7226khSSqcQSBJhSstCLb1ugE9UmK/S+wzlNnvEvsMXex3UTUCSdLLlTYikCTNYBBIUuGKCYKIuDwivhUR+yPiw71uTx0i4oyIeDAi9kXEYxHxvur5UyLif0TEt6s/67sLdo9ExHBE/HlE7KwenxkRD1Xn+08iYmWv29htEXFyRNwdEd+MiG9ExJsLOdfvr/5+PxoRd0XEK5p2viPi9oh4NiIebXtu1nMbLb9f9f2RiNi40O8rIggiYhjYCrwNeD3wnoh4fW9bVYtJ4IOZ+XpgC3Bj1c8PAw9k5gbggepx07wP+Ebb498Gfi8zzwHGget60qp6fRL4YmaeB7yBVv8bfa4j4nTgnwObM/MCYBh4N807358CLp/x3Fzn9m3AhurnBuCWhX5ZEUEAXAzsz8wDmTkBbAeu6nGbui4zn87Mh6vfX6T1H8PptPr66eqwTwO/1JsW1iMi1gL/ALi1ehzAzwN3V4c0sc+vBn4OuA0gMycy83kafq4rK4BXRsQK4ATgaRp2vjPzK8D3Zzw917m9CrgjW3YBJ0fEaQv5vlKC4HTgu22PD1XPNVZErAcuAh4C1mTm09VLzwBretSsuvwH4DeAqerxauD5zJysHjfxfJ8JfA/442pK7NaIOJGGn+vMfAr4HeBJWgHwArCX5p9vmPvcLvn/t1KCoCgRcRJwD/DrmfnX7a9la71wY9YMR8SVwLOZubfXbVlmK4CNwC2ZeRHwA2ZMAzXtXANU8+JX0QrCvwWcyMunUBqv2+e2lCB4Cjij7fHa6rnGiYjjaIXAnZn5uerp/zs9VKz+fLZX7avBJcAvRsQTtKb8fp7W3PnJ1dQBNPN8HwIOZeZD1eO7aQVDk881wC8A38nM72XmEeBztP4ONP18w9zndsn/v5USBLuBDdXKgpW0iks7etymrqvmxm8DvpGZv9v20g7gV6vffxX4wnK3rS6Z+ZHMXJuZ62md1/+ZmdcADwLvrA5rVJ8BMvMZ4LsRcW711FuAfTT4XFeeBLZExAnV3/fpfjf6fFfmOrc7gH9SrR7aArzQNoXUmcws4ge4AvgL4HHgo71uT019/Flaw8VHgK9VP1fQmjN/APg2cD9wSq/bWlP/R4Gd1e9nAV8F9gP/FTi+1+2rob8XAnuq8/2nwEgJ5xr4V8A3gUeBzwDHN+18A3fRqoEcoTX6u26ucwsErVWRjwP/h9aKqgV9n1tMSFLhSpkakiTNwSCQpMIZBJJUOINAkgpnEEhS4QwCqWYRMTq9K6rUjwwCSSqcQSBVIuJXIuKrEfG1iPij6h4HhyPi96r97x+IiNdUx14YEbuq/d8/37Y3/DkRcX9EfD0iHo6Is6uPP6nt3gF3VlfFEhGfqO4f8UhE/E6Puq7CGQQSEBE/A7wLuCQzLwSOAtfQ2tRsT2aeD3wZ+M3qLXcA/zIz/w6tqzmnn78T2JqZbwD+Lq2rQ6G1E+yv07ofxlnAJRGxGvhHwPnV5/ybenspzc4gkFreAmwCdkfE16rHZ9Ha2vpPqmP+M/Cz1b0ATs7ML1fPfxr4uYhYBZyemZ8HyMwfZuZL1TFfzcxDmTlFa+uP9bS2UP4hcFtEvB2YPlZaVgaB1BLApzPzwurn3Mz8rVmOW+yeLD9q+/0osCJb++dfTGvn0CuBLy7ys6UlMQiklgeAd0bEa+HH94ddR+vfyPSullcD/yszXwDGI+LvVc9fC3w5W3eFOxQRv1R9xvERccJcX1jdN+LVmXkv8H5at5uUlt2KYx8iNV9m7ouIjwFfioghWrs+3kjrhi8XV689S6uOAK1tgP+w+o/+APDe6vlrgT+KiJurz/jleb52FfCFiHgFrRHJB7rcLakj7j4qzSMiDmfmSb1uh1Qnp4YkqXCOCCSpcI4IJKlwBoEkFc4gkKTCGQSSVDiDQJIK9/8A+FGEVDGQUqQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss,'.')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('loss')\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "id": "07hbTNFbY_9U"
   },
   "outputs": [],
   "source": [
    "def momentum(x,y,epochs=100,learningrate=0.01,gamma=0.1):\n",
    "  loss=[]\n",
    "  learningrate=0.01\n",
    "  w=np.random.normal(0,0.01,9)\n",
    "  momentum=0\n",
    "  for i in range(epochs):\n",
    "    a=forward_propagation(x[0],y[0],w)\n",
    "    for i in a:\n",
    "      a[i]=0\n",
    "    for i in range(len(x)):\n",
    "      fp=forward_propagation(x[i],y[i],w)\n",
    "      for i in fp:\n",
    "        a[i]=fp[i]+a[i]\n",
    "    for i in a:\n",
    "      a[i]=a[i]/len(x)\n",
    "    loss.append(a['loss'])\n",
    "    b=backward_propagation(x[0],y[0],w,a)\n",
    "    for i in b:\n",
    "      b[i]=0\n",
    "    for i in range(len(x)):\n",
    "      bp=backward_propagation(x[i],y[i],w,a)\n",
    "      for i in bp:\n",
    "        b[i]=b[i]+bp[i]\n",
    "    for i in b:\n",
    "      b[i]=b[i]/len(x)\n",
    "    new_weights=[]\n",
    "    for vv in range(len(b)):\n",
    "      new_weights.append(b[f'dw{vv+1}'])\n",
    "    new_weights=np.array(new_weights)\n",
    "    #w=w-learningrate*new_weights\n",
    "    momentum=gamma*(momentum)+learningrate*new_weights\n",
    "    w=w-momentum\n",
    "  return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "8tc7KKVQZAB7"
   },
   "outputs": [],
   "source": [
    "loss=momentum(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 279
    },
    "id": "-gtGU5cOZAEe",
    "outputId": "3446a482-6394-40a4-bfb4-d80b060407f0"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAY90lEQVR4nO3df5Bd5X3f8fd39cMUJBsBjuIgLAlQTTCtMdqAXNJ4Ff+ocBmTYhxjMEkcVM10oHWM68Qee4hL8wf5HdKojlVBgx0GJQH/0KjUUChrTzpZLC1xXITBFjLCwlAcWFNkYu+u9ts/7ll6We6u7v44e+895/2a0Wjvvefe+zxzxH54nu9znhOZiSSpvvo63QBJUmcZBJJUcwaBJNWcQSBJNWcQSFLNLe10A2brlFNOyXXr1s3pvT/84Q854YQTFrZBPaCO/a5jn6Ge/a5jn2H2/R4eHv77zHxtq9d6LgjWrVvHvn375vTewcFBBgYGFrZBPaCO/a5jn6Ge/a5jn2H2/Y6IQ9O95tSQJNWcQSBJNWcQSFLNGQSSVHMGgSTVnEEgSTVXmyAYPjTCnsdGGT400ummSFJXqUUQDB8a4cqdQ9z57TGu3DlkGEhSk1oEwdDBZxkdnyCBsfEJhg4+2+kmSVLXqEUQbDr9ZJYv7aMPWLa0j02nn9zpJklS16hFEGxcu4rbtm7i0g3LuG3rJjauXdXpJklS1+i5vYbmauPaVbxwxnJDQJKmqMWIYKrhQyNsv/+ARWNJokYjgkmTK4hGxydYvrTPqSJJtVe7EcHkCqKJdAWRJEENg2ByBdGScAWRJEENp4YmVxANHXyWTaef7LSQpNqrXRBAIwwMAElqqN3U0FSuIJJUd7UcEUxyBZEk1XxE4AoiSap5ELiCSJJqPjXkCiJJqnkQgCuIJKnWU0OtuIpIUt3UfkTQzFVEkuqotBFBRNwSEc9ExEPTvB4R8ccRcSAivhER55XVlna5ikhSHZU5NfRnwJYZXr8I2FD82QZ8usS2tMVVRJLqqLSpocz8akSsm+GQS4DPZmYCQxFxYkS8LjOfKqtNx+IqIkl1FI3fwyV9eCMI9mTmOS1e2wPcmJl/XTy+D/iNzNzX4thtNEYNrF69euOuXbvm1J4jR46wYsWKOb23l9Wx33XsM9Sz33XsM8y+35s3bx7OzP5Wr/VEsTgzdwA7APr7+3NgYGBOnzM4OMhs3jt8aKQSo4PZ9rsK6thnqGe/69hnWNh+dzIIngROa3q8pniuK7iCSFJddPI6gt3ALxWrhzYBz3eyPjCVK4gk1UVpI4KIuB0YAE6JiMPAbwLLADLzT4G7gHcBB4AXgQ+W1Za5mFxBNDY+4QoiSZVW5qqh9x/j9QSuKev758sVRJLqoieKxZ3SvA9RVQrHkjSVQdAGC8eSqsxN59pg4VhSlRkEbXDrCUlV5tRQGywcS6oyg6BNFo4lVZVBMEsWjiVVjTWCWbJwLKlqDIJZsnAsqWqcGpolC8eSqsYgmAMLx5KqxCCYBwvHkqrAGsE8WDiWVAUGwTxYOJZUBU4NzYOFY0lVYBDMk4VjSb3OIFggFo4l9SprBAvEwrGkXmUQLBALx5J6lVNDC2Rq4Rhg+/0HrBdI6noGwQKaLBxbL5DUS5waKoH1Akm9xCAogfUCSb3EqaESWC+Q1EsMgpJYL5DUK5waKpn1AkndziAomfUCSd3OqaGSWS+Q1O0MgkVgvUBSN3NqaBFZL5DUjQyCRWS9QFI3cmpoEVkvkNSNDIJFZr1AUrdxaqhDrBdI6hYGQYdYL5DULZwa6hDrBZK6RalBEBFbgJuAJcDOzLxxyuuvB24FTiyO+Vhm3lVmm7qJ9QJJ3aC0qaGIWAJsBy4CzgbeHxFnTznsk8BfZuabgcuB/1xWe7qZ9QJJnVRmjeB84EBmHszMUWAXcMmUYxJ4dfHza4DvldiermW9QFInlTk1dCrw3abHh4ELphzzKeCeiPi3wAnA20tsT9eyXiCpkyIzy/ngiMuALZm5tXh8FXBBZl7bdMx1RRt+PyLeAtwMnJOZE1M+axuwDWD16tUbd+3aNac2HTlyhBUrVszpvYvlwMhRfmfvjxibgGV98Os/cxxnrloyr8/shX4vtDr2GerZ7zr2GWbf782bNw9nZn+r18ocETwJnNb0eE3xXLOrgS0Amfk3EXEccArwTPNBmbkD2AHQ39+fAwMDc2rQ4OAgc33vYtl//wHG81ESOJrw4xPXMjBw5rw+sxf6vdDq2GeoZ7/r2GdY2H6XWSPYC2yIiPURsZxGMXj3lGOeAN4GEBE/DRwHfL/ENnW9qfWCVccvZ/v9Bxg+NNLppkmqqNJGBJk5HhHXAnfTWBp6S2buj4gbgH2ZuRv4CPBfIuLDNArHv5JlzVX1iOZ6warjl3PDnv0uK5VUqlKvIyiuCbhrynPXN/38MHBhmW3oRZPXF2y//8ArlpUaBJIWmltMdDGXlUpaDG4x0cVcVippMRgEXc5tKCSVzamhHuE2FJLKYhD0iFb1guFDIy4tlTRvTg31iFb1AqeKJC0Eg6CHTNYLAJeWSlowTg31KK9AlrRQHBH0KK9AlrRQHBH0sI1rV3HN5jMZeXHUFUWS5swgqACniSTNh1NDFXCsaSJJmokjgopwmkjSXBkEFdNqmmjPY6NOE0mallNDFdNqmujHYxPseXzI1USSWnJEUEFTp4kSp4kkTc8gqLDJaaI+XE0kaXpODVXY5DTR7ffu5bxzzvKiM0ktOSKouI1rV3HxGctdTSRpWgZBTXjRmaTpODVUE+5NJGk6bY0IIuJDEfHqaLg5Ih6MiHeW3TgtrOkuOrvzwcOODqQaa3dE8KuZeVNE/AtgFXAV8DngntJaptJMThONjU+wpC+4Y/gw40cdHUh11W6NIIq/3wV8LjP3Nz2nHjM5TXTdO9/Ae/tPY/yoRWSpztoNguGIuIdGENwdESuBifKapbJNThNdet4ai8hSzbU7NXQ1cC5wMDNfjIiTgA+W1ywtFovIktodEbwFeDQzfxARHwA+CTxfXrO0mNy5VKq3doPg08CLEfEm4CPAY8BnS2uVOsJrDaR6andqaDwzMyIuAf4kM2+OiKvLbJgWn9NEUj21OyJ4ISI+TmPZ6H+LiD5gWXnNUqd4rYFUP+2OCN4HXEHjeoKnI+L1wO+W1yx1mtcaSPXR1oggM58GbgNeExEXAz/KTGsEFTbTtQaODqRqaWtEEBG/SGMEMEjjQrL/FBEfzcw7SmybOmzj2lVsXLuK4UMj3PngYUcHUkW1OzX0CeBnMvMZgIh4LXAvYBDUQHMR+Xs/+Adu/9oTL1tiahBIva3dYnHfZAgUnp3Fe1UBXoksVVe7I4IvR8TdwO3F4/cBd5XTJHUzl5hK1dNWEGTmRyPiPcCFxVM7MvML5TVL3WyydrD9/gOvWGI6dPBZNp1+soEg9ZC2b0yTmXcCd87mwyNiC3ATsATYmZk3tjjmF4FPAQn8XWZeMZvvUOe4xFSqhhmDICJeoPEL+hUvAZmZr57hvUuA7cA7gMPA3ojYnZkPNx2zAfg4cGFmjkTET8yhD+qQYxWRAUcIUg+YMQgyc+U8Pvt84EBmHgSIiF3AJcDDTcf8a2B7Zo4U3/fMKz5FXa3VEtPJIvKVO4esH0g9IDJb/Q//AnxwxGXAlszcWjy+CrggM69tOuaLwLdo1B6WAJ/KzC+3+KxtwDaA1atXb9y1a9ec2nTkyBFWrFgxp/f2ssXq94GRozzy3FHOOmkJjzx3lDu/PUbSWF526YZlXHzG8tLbMMlzXR917DPMvt+bN28ezsz+Vq91+ub1S4ENwACwBvhqRPyTzPxB80GZuQPYAdDf358DAwNz+rLBwUHm+t5etlj9bv6G4UMj7Hl86KURwnnnnMX+F0cXbZrIc10fdewzLGy/ywyCJ4HTmh6vKZ5rdhh4IDPHgO9ExLdoBMPeEtulReAyU6l3lHlR2F5gQ0Ssj4jlwOXA7inHfJHifyQj4hTgHwMHS2yTFpE7mUq9obQRQWaOR8S1wN005v9vycz9EXEDsC8zdxevvTMiHgaOAh/NTG+LVTEuM5W6W6k1gsy8iylXIGfm9U0/J3Bd8UcVNdMyUy9Ckzqv08Vi1YQ7mUrdyyDQonJ0IHUfg0CLztGB1F0MAnWMowOpOxgE6ihHB1LnGQTqCo4OpM4xCNQ1HB1InWEQqOs4OpAWl0GgruToQFo8BoG6mqMDqXwGgbpeO6OD6y9+IyOLuM21VCUGgXrGdKOD0bEJrv/SQ0xkvjRlJKl9BoF6SqvRQUQwkfmyKaPR50ZZuX7E0YHUBoNAPanVjW+ap4zGxifY8/iQBWWpDQaBetbk6ADgDT+58mVTRokFZaldBoEqYeqU0eiYy02ldpV5q0pp0U1OGV26YRnv7T+N8aPeIlM6FkcEqpyNa1fxwhnLWbl+jctNpTYYBKqs2Sw3NQxUZwaBKq2d5aZDB58FsKis2jIIVAvTLTddtrSPVccv58qdQ4yOW1RWPRkEqo1Wy003nX4yQwefZXR8wj2MVFsGgWqpORQAli/ts6is2jIIVHsWlVV3BoFE+3sYOWWkKjIIpCbH2sPIKSNVkUEgTTHTHkZOGamKDAJpBk4ZqQ4MAqkNThmpygwCqU1OGamqDAJpDpwyUpUYBNI8OGWkKjAIpHlyyki9ziCQFpBTRupFBoFUAqeM1EsMAqkks50yMhTUKaXeszgitkTEoxFxICI+NsNx74mIjIj+MtsjdcrGtau4ZvOZXHreGpYv7WNJQF/f/58ymgyF37/nUa7cOeR9lbWoShsRRMQSYDvwDuAwsDcidmfmw1OOWwl8CHigrLZI3WK6KSPrCOqkMqeGzgcOZOZBgIjYBVwCPDzluP8I/Dbw0RLbInWNVlNGM9URbtu6CfBWmipPZGY5HxxxGbAlM7cWj68CLsjMa5uOOQ/4RGa+JyIGgX+fmftafNY2YBvA6tWrN+7atWtObTpy5AgrVqyY03t7WR373Yt9PjBylEeeO8qz/5AMHh4naczdvnXNEv7X944yNgHL+uCKs5ZzZCw566QlnLlqycs+oxf7PV917DPMvt+bN28ezsyW0+8dKxZHRB/wB8CvHOvYzNwB7ADo7+/PgYGBOX3n4OAgc31vL6tjv3uxzwPF38OHRvibnUMv3VP5p049lfEnnyCB8Qn480fGigLz0VcUmHux3/NVxz7Dwva7zCB4Ejit6fGa4rlJK4FzgMGIAPhJYHdEvLvVqECqi+Y6wqbTTwZoeU1CqwvVpLkoMwj2AhsiYj2NALgcuGLyxcx8Hjhl8vFMU0NS3Uy9p3K7BebR50ZZuX7EOoJmpbQgyMzxiLgWuBtYAtySmfsj4gZgX2buLuu7pappt8A8Nj7BnseHvCZBs1JqjSAz7wLumvLc9dMcO1BmW6SqmOlCtcQL1TR7Xlks9bCpexuNjk20vFDNUNBMDAKpAiYLzLffu5fzzjmrZR3BUNB0DAKpIjauXcULZyxn4ILXt6wjHGvFkWFQXwaBVEHHKi67pYWaGQRSxc12SwunjOrHIJBqxK2x1YpBINXUse6mZijUh0Eg1Vw7W2MbCtVmEEiaVXHZUKgeg0DSy8wnFLx3Qm8yCCRNay7LUD//4GFGx12B1EsMAkltOVYoLFvaRwCj4xNOIfUYg0DSrLUKhdncO8FQ6C4GgaR5me29EwyF7mMQSFpQrkDqPQaBpNIYCr3BIJC0KAyF7mUQSFp0hkJ3MQgkddR8Q+HBx0ZZuX7EUJgHg0BS15hLKBydSPY8PuRIYR4MAkldqd1QSJw+mi+DQFLXmykURscm6Otrv6YwfGjEvZCmMAgk9ZSpoXD7vXs575yz2q4p3LBnv3shTWEQSOpZG9eu4oUzljNwwevb2hTvvz/0lHshtWAQSKqEdjbFu+ic17H38edcojqFQSCpcqbbFG/j2lVet9CCQSCp0qZuirdQF7NBdW7AYxBIqq25hsLSvoAIxo9Wo+hsEEgSs7wb29EEqnMNg0EgSVMcKxSWFCOCo0erMZVkEEjSDGa6G1tVppIMAklqU6vCM/T+VJJBIEnztFBTSbdt3QTwilFH2SFhEEjSAprrVNLY+AR3PniYzz94mNHxiUWdTjIIJKkks5lKWra0j4CXtsCYaTppcuSwUAwCSVpkM40a7nzw8IzTSWPjEwwdfJY3xsK1p9QgiIgtwE3AEmBnZt445fXrgK3AOPB94Fcz81CZbZKkbjJ11HDb1k0zTictW9rHptNP5oXvHF6wNpQWBBGxBNgOvAM4DOyNiN2Z+XDTYX8L9GfmixHxb4DfAd5XVpskqdsdazppskYw+J2F+84yRwTnAwcy8yBAROwCLgFeCoLMvL/p+CHgAyW2R5J61tSAWEiRmeV8cMRlwJbM3Fo8vgq4IDOvneb4PwGezszfavHaNmAbwOrVqzfu2rVrTm06cuQIK1asmNN7e1kd+13HPkM9+13HPsPs+7158+bhzOxv9VpXFIsj4gNAP/DWVq9n5g5gB0B/f38ODAzM6XsGBweZ63t7WR37Xcc+Qz37Xcc+w8L2u8wgeBI4renxmuK5l4mItwOfAN6amT8usT2SpBb6SvzsvcCGiFgfEcuBy4HdzQdExJuBzwDvzsxnSmyLJGkapQVBZo4D1wJ3A98E/jIz90fEDRHx7uKw3wVWAH8VEV+PiN3TfJwkqSSl1ggy8y7grinPXd/089vL/H5J0rGVtmqoLBHxfWCuF52dAvz9AjanV9Sx33XsM9Sz33XsM8y+32sz87WtXui5IJiPiNg33fKpKqtjv+vYZ6hnv+vYZ1jYfpdZLJYk9QCDQJJqrm5BsKPTDeiQOva7jn2Geva7jn2GBex3rWoEkqRXqtuIQJI0hUEgSTVXmyCIiC0R8WhEHIiIj3W6PWWIiNMi4v6IeDgi9kfEh4rnT4qI/xER3y7+Lu8u2B0SEUsi4m8jYk/xeH1EPFCc778otjmplIg4MSLuiIhHIuKbEfGWmpzrDxf/vh+KiNsj4riqne+IuCUinomIh5qea3luo+GPi75/IyLOm+331SIImm6ScxFwNvD+iDi7s60qxTjwkcw8G9gEXFP082PAfZm5AbiveFw1H6Kxlcmk3wb+MDPPBEaAqzvSqnLdBHw5M88C3kSj/5U+1xFxKvDvaNzQ6hwadz+8nOqd7z8Dtkx5brpzexGwofizDfj0bL+sFkFA001yMnMUmLxJTqVk5lOZ+WDx8ws0fjGcSqOvtxaH3Qr8QmdaWI6IWAP8S2Bn8TiAnwfuKA6pYp9fA/wccDNAZo5m5g+o+LkuLAX+UUQsBY4HnqJi5zszvwo8N+Xp6c7tJcBns2EIODEiXjeb76tLEJwKfLfp8eHiucqKiHXAm4EHgNWZ+VTx0tPA6g41qyx/BPw6MFE8Phn4QbHxIVTzfK+ncZ/v/1pMie2MiBOo+LnOzCeB3wOeoBEAzwPDVP98w/Tndt6/3+oSBLUSESuAO4Ffy8z/2/xaNtYLV2bNcERcDDyTmcOdbssiWwqcB3w6M98M/JAp00BVO9cAxbz4JTSC8KeAE3jlFErlLfS5rUsQtHWTnCqIiGU0QuC2zPx88fT/mRwqFn9X6d4PFwLvjojHaUz5/TyNufMTi6kDqOb5PgwczswHisd30AiGKp9rgLcD38nM72fmGPB5Gv8Gqn6+YfpzO+/fb3UJgmPeJKcKirnxm4FvZuYfNL20G/jl4udfBr602G0rS2Z+PDPXZOY6Guf1f2bmlcD9wGXFYZXqM0BmPg18NyLeUDz1NuBhKnyuC08AmyLi+OLf+2S/K32+C9Od293ALxWrhzYBzzdNIbUnM2vxB3gX8C3gMeATnW5PSX38WRrDxW8AXy/+vIvGnPl9wLeBe4GTOt3Wkvo/AOwpfj4d+BpwAPgr4FWdbl8J/T0X2Fec7y8Cq+pwroH/ADwCPAR8DnhV1c43cDuNGsgYjdHf1dOdWyBorIp8DPjfNFZUzer73GJCkmquLlNDkqRpGASSVHMGgSTVnEEgSTVnEEhSzRkEUskiYmByV1SpGxkEklRzBoFUiIgPRMTXIuLrEfGZ4h4HRyLiD4v97++LiNcWx54bEUPF/u9faNob/syIuDci/i4iHoyIM4qPX9F074DbiqtiiYgbi/tHfCMifq9DXVfNGQQSEBE/DbwPuDAzzwWOAlfS2NRsX2a+EfgK8JvFWz4L/EZm/lMaV3NOPn8bsD0z3wT8MxpXh0JjJ9hfo3E/jNOBCyPiZOBfAW8sPue3yu2l1JpBIDW8DdgI7I2IrxePT6extfVfFMf8OfCzxb0ATszMrxTP3wr8XESsBE7NzC8AZOaPMvPF4pivZebhzJygsfXHOhpbKP8IuDkiLgUmj5UWlUEgNQRwa2aeW/x5Q2Z+qsVxc92T5cdNPx8FlmZj//zzaewcejHw5Tl+tjQvBoHUcB9wWUT8BLx0f9i1NP4bmdzV8grgrzPzeWAkIv558fxVwFeycVe4wxHxC8VnvCoijp/uC4v7RrwmM+8CPkzjdpPSolt67EOk6svMhyPik8A9EdFHY9fHa2jc8OX84rVnaNQRoLEN8J8Wv+gPAh8snr8K+ExE3FB8xntn+NqVwJci4jgaI5LrFrhbUlvcfVSaQUQcycwVnW6HVCanhiSp5hwRSFLNOSKQpJozCCSp5gwCSao5g0CSas4gkKSa+3+XnwB8mNJL3AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss,'.')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('loss')\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "8IwEXOK-ZAJJ"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "HOGEbaqnZAM9"
   },
   "outputs": [],
   "source": [
    "def adam(x,y,epochs=100,learningrate=0.01,gamma=0.1,b1=0.1,b2=0.1):\n",
    "  loss=[]\n",
    "  lrchange=0\n",
    "  learningrate=0.01\n",
    "  w=np.random.normal(0,0.01,9)\n",
    "  momentum=0\n",
    "  for i in range(epochs):\n",
    "    a=forward_propagation(x[0],y[0],w)\n",
    "    for i in a:\n",
    "      a[i]=0\n",
    "    for i in range(len(x)):\n",
    "      fp=forward_propagation(x[i],y[i],w)\n",
    "      for i in fp:\n",
    "        a[i]=fp[i]+a[i]\n",
    "    for i in a:\n",
    "      a[i]=a[i]/len(x)\n",
    "    loss.append(a['loss'])\n",
    "    b=backward_propagation(x[0],y[0],w,a)\n",
    "    for i in b:\n",
    "      b[i]=0\n",
    "    for i in range(len(x)):\n",
    "      bp=backward_propagation(x[i],y[i],w,a)\n",
    "      for i in bp:\n",
    "        b[i]=b[i]+bp[i]\n",
    "    for i in b:\n",
    "      b[i]=b[i]/len(x)\n",
    "    new_weights=[]\n",
    "    for vv in range(len(b)):\n",
    "      new_weights.append(b[f'dw{vv+1}'])\n",
    "    new_weights=np.array(new_weights)\n",
    "    #w=w-learningrate*new_weights\n",
    "    momentum=gamma*(momentum)+learningrate*new_weights\n",
    "    #w=w-momentum\n",
    "    momentum=b1*momentum+(1-b1)*new_weights\n",
    "    lrchange=b2*lrchange+(1-b2)*new_weights**2\n",
    "    w=w-(learningrate/np.sqrt(lrchange/(1-b1)+10**-7)) * momentum/(1-b2)\n",
    "  return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "N8Uaj6xef342"
   },
   "outputs": [],
   "source": [
    "loss=adam(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 279
    },
    "id": "_Dnji1eqhHRS",
    "outputId": "eefc0cb6-e919-482e-c227-7e98021cff61"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAagUlEQVR4nO3df5RcdX3/8ed7d7MobIRNsCtfEpNAUlOgFbJbiF/auqk/Giw1raKgSNVCc04PObVqbfHooTbf7x/a3/SYWtNABUqzbYnWbZoKhbJ66nEh2dRaEqEsK4GNUDAsfFlR99f7+8e9kzOZzGxmZuezM/d+Xo9z9mTvnTszn8+5kFc+n/e9n2vujoiIxKut2Q0QEZHmUhCIiEROQSAiEjkFgYhI5BQEIiKR62h2A2p19tln++rVq+t67/e//33OOOOMxjYoA2Lsd4x9hjj7HWOfofZ+j4yMfM/dX13utcwFwerVqzlw4EBd7x0aGqK/v7+xDcqAGPsdY58hzn7H2Geovd9mdqTSa5oaEhGJnIJARCRyCgIRkcgpCEREIqcgEBGJnIJARCRy0QTByJEJ9j4+xciRiWY3RUSkpUQRBCNHJrh21zB7Hpvm2l3DCgMRkSJRBMHw2DGmZuZwYHpmjuGxY81ukohIy4giCDaet5zOjjbagCUdbWw8b3mzmyQi0jKiCILeVd3cdcNG3rFuCXfdsJHeVd3NbpKISMvI3FpD9epd1c1L53fSu6qbkSMTDI8dY+N5yxUKIhK9aIKgoFA4npqZo7OjTSMEEYleFFNDxQqF4zlX4VhEBCIMgkLhuN1UOBYRgQinhgqFY9UIREQSwYLAzG4DrgSedfeLyrxuwC3A24CXgQ+4+8FQ7SnWu6r7eACocCwisQs5IvgC8FngjgqvXwGsS38uAz6X/rloVDgWEQlYI3D3rwHPz3PIFuAOTwwDZ5nZOaHaU44KxyIizS0Wnws8VbQ9nu5bNCoci4iAuXu4DzdbDeytUCPYC3za3f893b4f+F13P+nJ9Ga2FdgK0NPT0zswMFBXeyYnJ+nq6jph3+jELI88P8v6Ze2s7W6v63NbXbl+512MfYY4+x1jn6H2fm/atGnE3fvKvdbMq4aOAiuLtlek+07i7juBnQB9fX3e399f1xcODQ1R+t7ST8pj8bhcv/Muxj5DnP2Osc/Q2H43MwgGgW1mNkBSJH7R3Z9uYntUPBaRKIW8fHQ3yT+4zzazceD3gCUA7v6XwD6SS0dHSS4f/WCotlSrXPFYQSAieRcsCNz9Pad43YEbQ31/PQrF4+mZORWPRSQa0d1ZPB/ddSwiMVIQlNBdxyISGwVBBSoci0gsolt9tFq661hEYqEgqEB3HYtILDQ1VEFp4RhgxwOjqheISO4oCOZRKByrXiAieaapoSqoXiAieaYgqILqBSKSZ5oaqoJuNBORPFMQVEk3molIXikIaqTCsYjkjWoENVLhWETyRkFQIxWORSRvNDVUIxWORSRvFAR1UOFYRPJEQbAAKhyLSB6oRrAAKhyLSB4oCBZAhWMRyQNNDS2AVigVkTxQECyQVigVkazT1FCDqF4gIlmlIGgQ1QtEJKs0NdQgqheISFYpCBpI9QIRySJNDQWgeoGIZImCIADVC0QkSzQ1FIAWphORLFEQBKKF6UQkK4IGgZltBm4B2oFd7v7pktdfC9wOnJUec5O77wvZpsWmwrGItLpgNQIzawd2AFcAFwDvMbMLSg77JPD37n4JcA3wF6Ha0ywqHItIqwtZLL4UGHX3MXefAgaALSXHOPCq9Pczge8GbE9TqHAsIq3O3D3MB5tdBWx29xvS7euAy9x9W9Ex5wD3At3AGcCb3X2kzGdtBbYC9PT09A4MDNTVpsnJSbq6uup670KMTszyyPOzrF/WDnD897Xd7Yvy/c3qdzPF2GeIs98x9hlq7/emTZtG3L2v3GvNLha/B/iCu/+xmb0BuNPMLnL3ueKD3H0nsBOgr6/P+/v76/qyoaEh6n3vQhS+8cR6weyi1Qua1e9mirHPEGe/Y+wzNLbfIaeGjgIri7ZXpPuKXQ/8PYC7fwN4BXB2wDY1leoFItKKQgbBfmCdma0xs06SYvBgyTFPAm8CMLOfIAmC5wK2qalULxCRVhRsasjdZ8xsG3APyaWht7n7ITPbDhxw90Hgo8BfmdmHSQrHH/BQRYsWoIXpRKQVBa0RpPcE7CvZd3PR74eBy0O2odVoYToRaTVaa6hJVC8QkVahIGgS1QtEpFU0+/LRaKleICKtQkHQRKoXiEgr0NRQC1C9QESaSUHQAlQvEJFm0tRQC1C9QESaSUHQIlQvEJFm0dRQi1G9QEQWm4KgxaheICKLTVNDLUb1AhFZbAqCFqR6gYgsJk0NtTDVC0RkMSgIWpjqBSKyGDQ11MJULxCRxaAgaHGqF4hIaJoaygjVC0QkFAVBRqheICKhaGooI0rrBYXpouJtEZF6KAgypFAvAFQzEJGG0dRQRqlmICKNoiDIqNKaQffpnex4YJSRIxPNbpqIZIymhjKquGbQfXon2/ce0jSRiNRFI4IM613VzY2b1jLx8pSmiUSkbgqCHNClpSKyEJoaygEtRSEiC6EgyIn5lqIQEZmPpoZyRpeVikitggaBmW02s0fNbNTMbqpwzLvN7LCZHTKzvw3ZnhiUu6x07+NTuqxURCoKNjVkZu3ADuAtwDiw38wG3f1w0THrgI8Dl7v7hJn9WKj2xKLcZaU/mp5j7xPDuqxURMoKOSK4FBh19zF3nwIGgC0lx/w6sMPdJwDc/dmA7YlG6WWljqaJRKSykMXic4GnirbHgctKjvlxADP7OtAOfMrdv1L6QWa2FdgK0NPTw9DQUF0NmpycrPu9WXTaC7N0GMy4027Gc099h9++dYz1y9pZ293e7OYFFdu5Loix3zH2GRrb72ZfNdQBrAP6gRXA18zsJ939heKD3H0nsBOgr6/P+/v76/qyoaEh6n1vFvUDl2yYYPd9+9lw0fqiu49ncz9NFNu5Loix3zH2GRrb75BTQ0eBlUXbK9J9xcaBQXefdvfvAP9NEgzSIL2rurny/E7dfSwiFYUMgv3AOjNbY2adwDXAYMkx/0jyD1fM7GySqaKxgG2Klu4+FpFKgk0NufuMmW0D7iGZ/7/N3Q+Z2XbggLsPpq+91cwOA7PAx9xd/1QNQHcfi0glQWsE7r4P2Fey7+ai3x34SPojgc1397HCQCReurM4Qrr7WESKVRUEZvYhM3uVJW41s4Nm9tbQjZMw9FAbESlW7dTQr7n7LWb2C0A3cB1wJ3BvsJZJMHqojYgUq3ZqyNI/3wbc6e6HivZJBumhNiJSUG0QjJjZvSRBcI+ZLQXmwjVLFoumiUSk2qmh64GLgTF3f9nMlgEfDNcsWSyaJhKRakcEbwAedfcXzOx9wCeBF8M1SxZTpWmiPQfHNToQiUC1I4LPAa83s9cDHwV2AXcAbwzVMFl8hWmi6Zk52tuMu0fGmZnV6EAk76odEcykN39tAT7r7juApeGaJc1QmCb6yFtfx7v6VjIzqyKySAyqDYKXzOzjJJeN/rOZtQFLwjVLmqUwTfSODStURBaJRLVTQ1cD7yW5n+AZM3st8IfhmiXNpiKySDyqGhG4+zPAXcCZZnYl8EN3vyNoy6TpdK+BSByqXWLi3cBDwLuAdwMPmtlVIRsmrUP3GojkW7VTQ58AfrrwTGEzezVwH3B3qIZJ69A0kUi+VVssbit5sPyxGt4rOaB7DUTyq9oRwVfM7B5gd7p9NSXPGZA46F4DkfypKgjc/WNm9k7g8nTXTnf/UrhmSasqnib67gs/YPdDT55QRFYQiGRP1U8oc/c9wJ6AbZGMKH7S2Z6D40zPzJ1QRNbjL0WyZd4gMLOXAC/3EsmTJl8VpFWSCSoii+TDvEHg7lpGQuZVGB3seGC07L0Gw2PHNEIQaXFBH14v8SguIhemia7dNawRgkgGKAikIYqniTaet5zhsWMnjRAUBCKtSUEgDVOYJiooHSGokCzSmhQEEoQKySLZobuDJRjdjSySDRoRSHC6G1mktSkIJLj57kbec3Bcl5iKNJmCQBZFubuRNToQaQ0KAllUGh2ItJ6gxWIz22xmj5rZqJndNM9x7zQzN7O+kO2R1lDuuciF0cEf3/so1+4aViFZZBEFGxGYWTuwA3gLMA7sN7NBdz9cctxS4EPAg6HaIq1JowOR1hByauhSYNTdxwDMbADYAhwuOe7/AJ8BPhawLdKiVDsQaT5zL7e4aAM+OHmm8WZ3vyHdvg64zN23FR2zAfiEu7/TzIaA33b3A2U+ayuwFaCnp6d3YGCgrjZNTk7S1dVV13uzLCv9Hp2Y5ZHnZzn2A2dofAYnmbt844p2lr+yjfXL2lnb3V7VZ2Wlz40WY79j7DPU3u9NmzaNuHvZ6femFYvNrA34E+ADpzrW3XcCOwH6+vq8v7+/ru8cGhqi3vdmWVb63Z/+OXJkgm/sGj4+Ovj6M87M7DSdHbNVjw6y0udGi7HfMfYZGtvvkEFwFFhZtL0i3VewFLgIGDIzgNcAg2b29nKjAomHagciiytkEOwH1pnZGpIAuAZ4b+FFd38ROLuwPd/UkMRHtQORxRMsCNx9xsy2AfcA7cBt7n7IzLYDB9x9MNR3S35odCASXtAagbvvA/aV7Lu5wrH9Idsi2aXRgUhYurNYMkOjA5EwFASSKdWODkSkegoCyaRTjQ6mnp9i6ZoJjQ5EqqAH00hmzbdm0Z7HprVmkUiVNCKQzCs3OnBUOxCploJAcqG0djA1fXLt4OYrL2Ti5SmFgkgJBYHkSmF0sPu+/Zy27JzjtYOp6Tlu/vLDzLnrclOREqoRSO70rurmyvM7T6gdtLUZc+4nFJR3PDCqGoIIGhFIjhXXDrpP72T73kO6GU2kDAWB5FqhdgDwutcsLXu56fDYMQAVlSVaCgKJRrmb0ZZ0tNF9eifX7hpmakYjBImTgkCiUzxltPG85QyPHWNqZk7LVUi0FAQSpeIpI4DOjray9QNdcioxUBBI9CotV6FLTiUWCgIRytcPzE6+5FRTRpJHCgKRItVecqopI8kTBYFIiVNdcqopI8kbBYHIPDRlJDFQEIhUQXcpS54pCESqVM1dyhodSBYpCETqUM0jM1VQlqxQEIgsgO5BkDxQEIgskArKknUKApEG0T0IklUKApEG0j0IkkUKApFANGUkWaEgEAlMU0bS6hQEIotAU0bSyoI+vN7MNpvZo2Y2amY3lXn9I2Z22My+ZWb3m9mqkO0RaQW9q7q5cdNa3rFhBZ0dbbQbtLWdPGW044FRRo5MNLu5EoFgIwIzawd2AG8BxoH9Zjbo7oeLDvsPoM/dXzaz3wD+ALg6VJtEWkkty1aAnqks4YScGroUGHX3MQAzGwC2AMeDwN0fKDp+GHhfwPaItJxql6344sFxPVNZggkZBOcCTxVtjwOXzXP89cC/lHvBzLYCWwF6enoYGhqqq0GTk5N1vzfLYux3Vvt8ocFpzNJhMOPQbvD00e/yo+k5nKSW8Nl/epDlr2xj/bJ21na3n/D+rPZ7IWLsMzS23y1RLDaz9wF9wBvLve7uO4GdAH19fd7f31/X9wwNDVHve7Msxn5nuc/9wCUbJo5PBQF8Y9fw8Wmjrz/jzMxO09kxe9KVRlnud71i7DM0tt8hg+AosLJoe0W67wRm9mbgE8Ab3f1HAdsjkhnFU0ZAVesZ3XzlhRx8fIqlayY0dSQ1CXnV0H5gnZmtMbNO4BpgsPgAM7sE+Dzwdnd/NmBbRDLtVFcaFUJhz2PTXLtrWFcbSU2CjQjcfcbMtgH3AO3Abe5+yMy2AwfcfRD4Q6AL+AczA3jS3d8eqk0iWVfpSqPCHcuO7liW2gWtEbj7PmBfyb6bi35/c8jvF8mjclcaFUJhalp3LEvtWqJYLCL1KQ2F3fft57Rl58xbR1AoSCkFgUhO9K7q5qXzO1m6ZkXZRe60jIVUoiAQyZlT1RG08qmUUhCI5NB8dQStfCqlFAQiOVfryqcKhfgoCEQicqqH5SgU4qQgEIlQNXUEFZfjoSAQidSp6ggqLsdDQSAiKi5HTkEgIidQcTk+CgIRqajW4rKeppZNCgIROaVablIrfpqaRgvZoCAQkaqcqo6wpKMNA6Zm5jSFlDEKAhGpWblQKDxNTfcnZI+CQEQWpNLT1Kq9P0Gh0HwKAhFpqFruT1AotAYFgYgEo1DIBgWBiCwKhULrUhCIyKJTKLQWBYGINNVCQ+Hg41MsXTOhUFgABYGItIx6QmF2ztn7xPAJIwXQ3c21UBCISEuqNhScE0cKHW0GZlogrwYKAhFpefOFwtT0HG1tRUtdzDpwckBoLaTKFAQikimlobD7vv1suGj9CUtmY8bsbG1rIUG8AaEgEJHM6l3VzUvnd9J/2WtPWuqilrWQ5ptOKnxWngNCQSAiuVC61EUtayFVmk6Kpd6gIBCRXKtmLaSK00nz1BvyNGpQEIhIVOZbObXagKhnWqn491YLi6BBYGabgVuAdmCXu3+65PXTgDuAXuAYcLW7PxGyTSIiBdVOJ837QJ4qppVqqUGMHJlY9MAIFgRm1g7sAN4CjAP7zWzQ3Q8XHXY9MOHua83sGuAzwNWh2iQiUo1TBUSt00q11CC27z206Fc1hRwRXAqMuvsYgJkNAFuA4iDYAnwq/f1u4LNmZu7uAdslIlKXeqeVqgqLmTn+5eGnq7qqqXA/RKOEDIJzgaeKtseByyod4+4zZvYisBz4XsB2iYgsWC3TStWExZKONq646Bz2P/H8vNNP0zNzDI8d40JrXF8s1D++zewqYLO735BuXwdc5u7bio55OD1mPN1+PD3meyWftRXYCtDT09M7MDBQV5smJyfp6uqq671ZFmO/Y+wzxNnvLPZ5dGKWR56fZf2ydoDjv6/tbj/+WtcS428fmWJmDtoMMJibg442+J2ffgWvWfKDmvq9adOmEXfvK/dayBHBUWBl0faKdF+5Y8bNrAM4k6RofAJ33wnsBOjr6/P+/v66GjQ0NES9782yGPsdY58hzn5nsc/9Vb72S0WFYzixRtDIfocMgv3AOjNbQ/IX/jXAe0uOGQTeD3wDuAr4N9UHREQSlaafGi1YEKRz/tuAe0guH73N3Q+Z2XbggLsPArcCd5rZKPA8SViIiMgiCnofgbvvA/aV7Lu56PcfAu8K2QYREZlfW7MbICIizaUgEBGJnIJARCRyCgIRkcgFu6EsFDN7DjhS59vPJs67lmPsd4x9hjj7HWOfofZ+r3L3V5d7IXNBsBBmdqDSnXV5FmO/Y+wzxNnvGPsMje23poZERCKnIBARiVxsQbCz2Q1okhj7HWOfIc5+x9hnaGC/o6oRiIjIyWIbEYiISAkFgYhI5KIJAjPbbGaPmtmomd3U7PaEYGYrzewBMztsZofM7EPp/mVm9q9m9lj65+I8EXsRmVm7mf2Hme1Nt9eY2YPp+f47M+tsdhsbzczOMrO7zewRM/u2mb0hknP94fS/74fNbLeZvSJv59vMbjOzZ9OHdxX2lT23lvjztO/fMrMNtX5fFEFgZu3ADuAK4ALgPWZ2QXNbFcQM8FF3vwDYCNyY9vMm4H53Xwfcn27nzYeAbxdtfwb4U3dfC0wA1zelVWHdAnzF3dcDryfpf67PtZmdC/wm0OfuF5EscX8N+TvfXwA2l+yrdG6vANalP1uBz9X6ZVEEAXApMOruY+4+BQwAW5rcpoZz96fd/WD6+0skfzGcS9LX29PDbgd+uTktDMPMVgC/COxKtw34eeDu9JA89vlM4OdInumBu0+5+wvk/FynOoBXpk81PB14mpydb3f/GskzWopVOrdbgDs8MQycZWbn1PJ9sQTBucBTRdvj6b7cMrPVwCXAg0CPuz+dvvQM0NOkZoXyZ8DvAHPp9nLgBXefSbfzeL7XAM8Bf51Oie0yszPI+bl296PAHwFPkgTAi8AI+T/fUPncLvjvt1iCICpm1gXsAX7L3f9f8Wvpo0Bzc82wmV0JPOvuI81uyyLrADYAn3P3S4DvUzINlLdzDZDOi28hCcL/BZzByVMoudfocxtLEBwFVhZtr0j35Y6ZLSEJgbvc/Yvp7v8pDBXTP59tVvsCuBx4u5k9QTLl9/Mkc+dnpVMHkM/zPQ6Mu/uD6fbdJMGQ53MN8GbgO+7+nLtPA18k+W8g7+cbKp/bBf/9FksQ7AfWpVcWdJIUlwab3KaGS+fGbwW+7e5/UvTSIPD+9Pf3A19e7LaF4u4fd/cV7r6a5Lz+m7tfCzwAXJUelqs+A7j7M8BTZva6dNebgMPk+FynngQ2mtnp6X/vhX7n+nynKp3bQeBX06uHNgIvFk0hVcfdo/gB3gb8N/A48IlmtydQH3+GZLj4LeCb6c/bSObM7wceA+4DljW7rYH63w/sTX8/D3gIGAX+ATit2e0L0N+LgQPp+f5HoDuGcw38PvAI8DBwJ3Ba3s43sJukBjJNMvq7vtK5BYzkqsjHgf8iuaKqpu/TEhMiIpGLZWpIREQqUBCIiEROQSAiEjkFgYhI5BQEIiKRUxCIBGZm/YVVUUVakYJARCRyCgKRlJm9z8weMrNvmtnn02ccTJrZn6br399vZq9Oj73YzIbT9d+/VLQ2/Fozu8/M/tPMDprZ+enHdxU9O+Cu9K5YzOzT6fMjvmVmf9SkrkvkFAQigJn9BHA1cLm7XwzMAteSLGp2wN0vBL4K/F76ljuA33X3nyK5m7Ow/y5gh7u/HvjfJHeHQrIS7G+RPA/jPOByM1sO/ApwYfo5/zdsL0XKUxCIJN4E9AL7zeyb6fZ5JEtb/116zN8AP5M+C+Asd/9quv924OfMbClwrrt/CcDdf+juL6fHPOTu4+4+R7L0x2qSJZR/CNxqZu8ACseKLCoFgUjCgNvd/eL053Xu/qkyx9W7JsuPin6fBTo8WT//UpKVQ68EvlLnZ4ssiIJAJHE/cJWZ/Rgcfz7sKpL/RwqrWr4X+Hd3fxGYMLOfTfdfB3zVk6fCjZvZL6efcZqZnV7pC9PnRpzp7vuAD5M8blJk0XWc+hCR/HP3w2b2SeBeM2sjWfXxRpIHvlyavvYsSR0BkmWA/zL9i34M+GC6/zrg82a2Pf2Md83ztUuBL5vZK0hGJB9pcLdEqqLVR0XmYWaT7t7V7HaIhKSpIRGRyGlEICISOY0IREQipyAQEYmcgkBEJHIKAhGRyCkIREQi9/8BW6BvJ1bRTIIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(loss,'.')\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('loss')\n",
    "plt.grid()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 279
    },
    "id": "TvktwilOh1vY",
    "outputId": "ece67933-fd98-435d-e481-a265c1445a4a"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de3xU5bXw8d+ayYSAIESQlAqSoFwEkUAi4lFOg1rFXoS2Wlov1ar4OeeIvdlafT2vpZ7W13q0p2o9bZGqaFW0UBWtVkSJ0haEBMNVWhGCRBEwAoICc1vvHzMTJ2EmmUlmZy57ff3wyeyZvfc8DxuzPs+znouoKsYYY9zLk+0CGGOMyS4LBMYY43IWCIwxxuUsEBhjjMtZIDDGGJcrynYB0jVgwAAtLy/v1LUff/wxRx11VGYLlAfcWG831hncWW831hnSr3d9ff0Hqnpsos/yLhCUl5dTV1fXqWtra2upqanJbIHygBvr7cY6gzvr7cY6Q/r1FpFtyT6zriFjjHE5CwTGGONyFgiMMcbl8i5HYIzJH4FAgKamJg4dOuTYd/Tt25c333zTsfvnqmT1LikpYfDgwfh8vpTvZYHAGOOYpqYm+vTpQ3l5OSLiyHfs37+fPn36OHLvXJao3qpKc3MzTU1NVFRUpHwv6xoyxjjm0KFD9O/f37EgYFoTEfr37592C8w1gaBhVwOL9y2mYVdDtotijKtYEOhenfn7dkUgaNjVwMzFM3lu73PMXDzTgoExxsRxRSCo21mHP+RHUQLhAHU7OzchzRjjXrfccgtLliwBoKampmVia3l5OR988EE2i9ZlrkgWV5dVU+wtxh/y4/P4qC6rznaRjDF55tZbb812ERzjihZB5cBKfnjKLzkhNJUfnvJLKgdWZrtIxpgk6rft4b6lm6nfticj97vxxhu57777Wo5nz57Nz372M84++2wmTJjA2LFjeeaZZwBobGzkpJNOYubMmYwZM4Zzzz2XgwcPAnDFFVewYMGCdr9r+vTpVFVVMWbMGObMmZOR8ncHVwSC+m17eOqPq5n0duTnE2uXMXfdXMsVGJNj6rft4ZK5K7hr8T+4ZO6KjASDGTNm8OSTT7YcP/nkk1x++eU89dRTrF69mqVLl3L99dcT27b3rbfe4tprr2XDhg3069ePhQsXpvxdDzzwAPX19dTV1XHPPffQ3Nzc5fJ3B1d0DW19YykPen6GzxOkvsdz/NsbZYQJU+wt5v5z77cWgjE5YsWWZvzBMGGFQDDMii3NVA0t7dI9x48fz65du3jvvffYvXs3paWlfOYzn+H73/8+r732Gh6Ph3fffZedO3cCUFFRQWVl5HdCVVUVjY2NKX/XPffcw1NPPQXA9u3beeutt+jfv3+Xyt8dXBEITvduxEeQIgnT0LOIMEHC0JI4tkBgTG6YNKw/xUUeAsEwviIPk4Zl5pfoRRddxIIFC3j//feZMWMGjz76KLt376a+vh6fz0d5eXnL2PsePXq0XOf1elu6hjpSW1vLkiVLWL58Ob169aKmpsbRGdWZ5IquoeMqz8VTVEwIDxP9IYrEh+DBK0WWODYmh1QNLeXRqyfxg3NH8ujVk7rcGoiZMWMG8+fPZ8GCBVx00UXs27ePgQMH4vP5WLp0Kdu2JV2hOWX79u2jtLSUXr16sWnTJlasWJGBkncPV7QIGDIRzxXPsuWVh+k5ejqfLN5MuHgzAf+JhA4OzXbpjDFxqoaWZiwAxIwZM4b9+/dz3HHHMWjQIC655BK+/OUvM3bsWKqrqxk1alSXv2Pq1Kn89re/5aSTTmLkyJFMmjQpAyXvHu4IBABDJvLO0E/YcGAwoz9ex2mffMRKPcjTb/6NNz56n+qyausiMqaArVu3ruX1gAEDWL58ecLz1q9f3/L6hz/8Ycvrhx56qOV1bW1ty+v4HMILL7zQ9YJmgXsCQdTZvRu50ncbPoLU9+jJf+z6DMGdIUscG2NcyxU5gnijDq2hxBNJHK/tVURQA4QJ24xjY4xrORYIROQBEdklIuuTfC4ico+IbBaRtSIywamytFI+GfH2APFyqj9siWNjjOs52TX0EPBr4OEkn58PDI/+OQ34TfSns4ZMhMsXQeMySkrGcdziV+hXsoa9h8ZZ4tgY40qOBQJVfU1Eyts5ZRrwsEam860QkX4iMkhVdzhVphZDJsKQiax/+k884b8fnz9IgJX8+Y1qvD2HUbezzpLHxhjXyGay+Dhge9xxU/Q95wNBVPxEMzRI3/BrzFz8//CH/JY8Nsa4Rl6MGhKRa4BrAMrKyloN3UrHgQMHWl17dHgAZZ4iQhoETxF/D3zM4dBhFOVwyM+Ty59kb9+9GahBdrWttxu4sc6Qe/Xu27cv+/fvd/Q7QqGQ49+RaWvXrmXHjh2cd955nb5He/U+dOhQWv8OshkI3gWGxB0Pjr53BFWdA8wBqK6u1pqamk59YW1tLa2vrYEJE6BxGZRPZtSew4Tr60GCqHoZV3E+NadM7tR35ZIj61343FhnyL16v/nmm47vJ5yPexa/9dZb1NXVceGFF3b6Hu3Vu6SkhPHjx6d8r2wOH10EfCs6emgSsK9b8gNtDZkIk6+HIRP5oHkQx28/j/EfDGHo9vP4oHlQtxfHGNfbvhKW3RX5mQGNjY2MGjWKK664ghEjRnDJJZewZMkSzjjjDIYPH87KlSv58MMPmT59OqeccgqTJk1i7dq1QGTJ6ssvv5zJkyczdOhQ/vSnP3HDDTcwduxYpk6dSiAQAKC+vp7Pfe5zVFVVcd5557FjR+RXWU1NDT/+8Y+ZOHEiI0aMYNmyZfj9fm655RaeeOIJKisreeKJJ5g9ezZ33nlnS5lPPvlkGhsb2y17ZWUlK1dm5u/IyeGjjwPLgZEi0iQiV4nIv4nIv0VPeR7YAmwG7gf+w6mypOrs3o0sDM3lwf1/Z0FoLr79f+Gqp3/BE2uXZbtoxrjD9pUw7wJ45eeRnxkKBps3b+b6669n06ZNbNq0iccee4y//vWv3Hnnndx222385Cc/Yfz48axdu5bbbruNb33rWy3Xvv3227zyyissWrSISy+9lClTprBu3Tp69uzJn//8ZwKBANdddx0LFiygvr6eK6+8kptvvrnl+mAwyMqVK/nVr37FT3/6U4qLi7n11luZMWMGDQ0NzJgxo1Nl//nPf85tt92Wkb8fJ0cNfbODzxW41qnv74xRh9agniCiYdb18PK/zfcTFOX1+ieAu5lRAN1ExuS0xmUQ8oOGIj8bl0Va7V1UUVHB2LFjgci6Q2effTYiwtixY2lsbGTbtm0t+w6cddZZNDc389FHHwFw/vnn4/P5GDt2LKFQiKlTpwK0XPuPf/yD9evX8/nPfx6I9N0PGvRpb8JXv/pVIP0lrTsq++jRozt1v0TyIlncbWKTzUJ+VvYsISSKiKIEWbzl7xYIjHFa+WTwFkeCgLc4cpwB8UtLezyelmOPx0MwGMTn83V4rcfjwefzISKtrlVVxowZk3Ttotj1Xq+XYDCY8JyioiLC4XDLcfzy1R2VPRNct8REu2KTzc66mc+M/09EPYiCRz2cO+xfsl06Ywpf3P+DXL4oI62BVEyePJlHH30UiCTcBwwYwNFHH53StSNHjmT37t0tgSAQCLBhw4Z2r+nTp0+rET/l5eWsXr0agNWrV7N169bOVKPTrEXQVnSy2bTtKzl+xW7qir1U+0P0PHyY+5ZuZtKw/hlfItcYEyf6/2B3mj17NldeeSWnnHIKvXr1Yt68eSlfW1xczIIFC/jOd77Dvn37CAaDfO9732PMmDFJr5kyZQq33347lZWV3HTTTXzta1/j4YcfZsyYMZx22mmMGDEiE9VKmcT26cwX1dXVWlfXucXh0hpat+yuSMJKQ6h4ualoKk8VfxaP/0T+cOmMvAoGuTaksDu4sc6Qe/V+8803Oemkkxz9jnwcPpoJ7dU70d+7iNSrasIF1axrKJlYX6V4qe/Rk+c/uxHfsYvxfnYOT7/5t2yXzhhjMsa6hpKJW5zupQO70Z3PI6JAkKJe3dt/Z4wxTrIWQXuik83OH3IKPTSMV5UeGqZSe3Lf0s3Ub9uT7RIaY0yXWYsgBZV7djB35wfU9fBRfTjA/Pde4aniTfz67/mXLzDGmLasRZCK8slUBuHqjz4miNfyBcaYgmKBIBVxY5tfGncZKmFEFJEge8KbrJvIGJPXLBCkKkm+4J03PuSuxf/gkrkrLBgYk6ceeughZs2ale1iZI3lCNIUny+oOhRgycEPWKsQCIZZsaXZ8gXGmLxjLYJ0xeULxgWVxp4h/vXY/2VM779R2qvYuomM6aKGXQ3MXTeXhl0NGbvn9OnTqaqqYsyYMcyZMweABx98kBEjRjBx4kT+9rdPc33PPvssp512GuPHj+ecc85h586dQOpLUucjaxGkK25+wVo9zOtvP0ZAwKfbmL8E1h84g+IiD49ePclaB8akqWFXAzMXz8z4drEPPPAAxxxzDAcPHuTUU0/li1/8Ij/5yU+or6+nb9++TJkypWUjlzPPPJMVK1YgIsydO5c77riDu+66C4gsSb106VI2btzI6aefzsKFC7njjjv4yle+wp///GemT5/e5bJmgwWCzoiuhVK36HICAmERgij9StYQ3n+GdRMZ00l1O+vwh/yECRMIB6jbWZeRQHDPPffw1FNPAbB9+3YeeeQRampqOPbYYwGYMWMG//znPwFoampixowZ7NixA7/fT0VFRct9OlqSOl9Z11AXVA87j2IFrypFCnsPjcMr4CvyWDeRMZ1QXVZNsbcYr3jxeXxUlyVcGicttbW1LFmyhOXLl7NmzRrGjx/PqFGjkp5/3XXXMWvWLNatW8fvfve7hEtCJ1uSOl9Zi6ALKk++mPuBui0vUj3sPLY1f0J940MMLv0ctz4H/mDYuomMSUPlwEruP/d+6nbWUV1WnZHWwL59+ygtLaVXr15s2rSJFStWcPDgQV599VWam5s5+uij+eMf/8i4ceNazj/uuOMA0lqFNJ9ZIOiiypMvpvLki2lY/xg/2/Ir/AK+5jcZ5vsyGwNnWjeRMWmqHFiZkQAQM3XqVH77299y0kknMXLkSCZNmsSgQYOYPXs2p59+Ov369aOy8tPvmz17NhdddBGlpaWcddZZ3b43QDZYIMiQui0v4o/LFwzsvZZ/HDwTX5GHScP6Z7t4xrhWjx49eOGFF454v6amhm9/+9tHvD9t2jSmTZt2xPuzZ89udXzgwIGkn+UbyxFkSHy+wKfwjRET+MOov/H0BZEt8CxfYIzJVdYiyJBW+YJ+I6l87R4I+Qm/83su9v8fVgZPtHyBMSYnWYsggypPvpirL5hHpfSgoQjmHn0Ua4qEKt1AODr7eOHqJmsdGFfJt10Q811n/r6tReCAhtJBzCwbEEkcK5z2XohrDz7DKkazoN5DMGSjiYw7lJSU0NzcTP/+/VuGWhrnqCrNzc2UlJSkdZ0FAgfU6Sf4PR7CKEERKnu/wlWBvQTFxzcP30S9jrDRRMYVBg8eTFNTE7t373bsOw4dOpT2L75CkKzeJSUlDB48OK17WSBwQGRSTA8C4QA+4NSDB/EQxkeQM4o20RAY0WrS2aRh/S0gmILk8/lazcx1Qm1tbcvyEG6SyXpbIHBAq0kx0gvev4G5/fpS7Q/zhSkX0eNAOaW9irn1uQ026cwYk3UWCBwSmxTTsKuBmYMG4g8HKPb4uL/nLq49tIYFOyrwB3u0JJGtm8gYky02ashhdTvr8IdDhIFAOETdyzfDKz/nq+v+nYlFm21tImNM1jnaIhCRqcDdgBeYq6q3t/n8eGAe0C96zo2q+ryTZepusUW0YvmC6oMHQUN4wnDXxP083XukdRMZY7LKsRaBiHiB+4DzgdHAN0VkdJvT/hN4UlXHA98A/tep8mRLLF8wa/ws7p9wA5VBQLzgLea4QYO5tugZinfU4Q+GW3UTGWNMd3GyRTAR2KyqWwBEZD4wDdgYd44CR0df9wXec7A8WRO/iFYDcbOP/3IjhPx81eNjQdH/YVXwRBtNZIzpduLUrD8RuRCYqqpXR48vA05T1Vlx5wwCFgOlwFHAOapan+Be1wDXAJSVlVXNnz+/U2U6cOAAvXv37tS1mbD18Fbu3XkvQQ3iQ/j9e+9TefgQYTxsOuYcGkMDeO+oMdzRWEEgDD4P3HBqCSeWerv0vdmudza4sc7gznq7sc6Qfr2nTJlSr6oJN3jI9qihbwIPqepdInI68IiInKyq4fiTVHUOMAegurpaa2pqOvVltbW1dPbaTNi8bjOh90MoSgihrlcvKv0BPB4vo/e9yuhwkMD+Il7gJuoZQUhhCwM5rD271DrIdr2zwY11BnfW2411hszW28lA8C4wJO54cPS9eFcBUwFUdbmIlAADgF0OlitrWiWOPT6qz/457NkB+5qgfh5oiCJomXTm9QgL6ptsSQpjjKOcHD66ChguIhUiUkwkGbyozTnvAGcDiMhJQAng3Fz0LGuVOD73fhg4mrn9+tJQfip4i0G8iLeYr555Cn8Y9Td+cNI+giFLIhtjnOVYi0BVgyIyC3iRyNDQB1R1g4jcCtSp6iLgeuB+Efk+kcTxFVrgSxW2mmi2eCb+kJ9ibzH3T/8llXt2QM/+lP/lRspDfk7z+Ki1JLIxxmGO5giicwKeb/PeLXGvNwJnOFmGXFW3sw5/yE+YMIFwgDr9hMrJ18OyuyDkt7kGxphuk+1ksWsdkS8oiybzyydHuolC/k/nGhx8JuGSFAArtjRbC8EY0yUWCLKk1cJ00SAwd91cqsuqqbx8ETQug579oZ25BpfMXWEtBGNMl1kgyKKk+YJz70/YTXTv6E1s9u+kdPRZvHzA32o28sLVTdY6MMZ0igWCHHBEvmBnXWQmcnw3kcfLsZsXcmw4CNsfgPP+wL1FHgLBsA0zNcZ0ia0+mgNi+QKveFvnC4ZMhMsXwVk3w/hLIRwEDUHIz6hDa3j06kn84NyRXFQ9xIaZGmM6zVoEOaDdfMGQiZGAsH0lNDzekkSmfDJVvEVV0TI2DR7HwtWR1oENMzXGpMsCQY5oN18wsPLT1kHjskiXEcC8CyKtA28xT1/wB15OsvOZMca0xwJBjkmaL4BIMBgyMfI6LpFMyM+onc8xqu+QI4aZLlzdhP9DP30q9ljrwBiTkAWCHJN0fkFbbRLJvPEYhIOthpnGksiBYJjnGldYEtkYk5AFghzTbr4g1jKA1l1FcYvWxc9Gfm/vQR5f+Q6KDTE1xiRngSAHdZgviEmSSI7NRo4lkf0BG2JqjEnOAkEOazdfEC++dRA3GzmWRJ67NkyPYwbx+Mp3bAKaMeYIFghyWMr5Avi0dZAgifwfRUH8g6e3DDG11oExJp4FghyWcr4gXoIkckUogGxf0DLENJY7sNaBMQYsEOS8lPMFMQmSyEK41RDT+Alo1jowxlggyBMp5wvgiCRyOHgYT9wQ0/gJaNY6MMZYIMgTifIFDbsaWrqN2msdNL7yMMP6+1qGmFrrwBgTzwJBnkiUL0i1q+idoZ8w7IRenw4xtdaBMSaOBYI8EssXQCRpnHJXESSdgBZbyXTUlHOo37aHhaubrHVgjMtYIMhTaQ0tjUm2kmnP/rDsLqrKJ/Po1ZNYsaXZWgfGuIgFgjzVqaGlMUkmoOEtpuryRVRNmdhu6+CWL41hzyd+CwrGFAgLBHks7aGl8ZJMQGPNY9C4LGnrwB8Ic8sz6wmrWpeRMQXCAkEBSGtoaVtJVjFN1joQEcKq1mVkTAGxQFAA2uYL+hb37Vw3UZskcqLWQWzjG0soG1M4LBAUgPh8Qd/ivtyx6o5W3UQdSpREbts6mHo7VUXN8NnJjLSEsjEFxQJBgYjlCxINKz2RE1O7SbLWQfAwPH89qKacULbWgTH5wwJBgUk0rHRv897Ub5CodSACGo78SSGhHAiGWbGlGcBaCMbkAQsEBSbRsNLF+xbTb1e/1BPIkHyIaQoJZV+Rh9JexVwydwX+oA05NSbXORoIRGQqcDfgBeaq6u0Jzvk6MBtQYI2qXuxkmdyg7bDSw6HDvLT4pdSGlcaLtQ4AykannFCeNKw/K7Y04w+GbcipMXnAsUAgIl7gPuDzQBOwSkQWqerGuHOGAzcBZ6jqHhEZ6FR53Cg2rFTR9IeVtpVKQjnaOogpLvLYkFNj8oCTLYKJwGZV3QIgIvOBacDGuHNmAvep6h4AVd3lYHlcJ5Yv8If86Q8rTSaF4aaUT6Zq6MSUhpxal5Ex2Seq6syNRS4Epqrq1dHjy4DTVHVW3DlPA/8EziDSfTRbVf+S4F7XANcAlJWVVc2fP79TZTpw4AC9e/fu1LX5auvhrWz4aAOlJaUs3LOQoAYpkiKuK7uOih4VXbr30fs2MW7N/0XCQVQ8CIJoiLCniDXj/ouP+o5qOXfznhCbPgzRfFCpbQqigAAegbCCzwM3nFrCiaXerlU4yo3PGtxZbzfWGdKv95QpU+pVNeGiZNlOFhcBw4EaYDDwmoiMVdVWw1xUdQ4wB6C6ulpramo69WW1tbV09tp8VUMNtbW1bO6/mdCHIRQlTJjQZ0PUjK3p8t2ZMKF164AwXg0x4ZiPobxXSwuhJpprqN+2h+VzV7TqMlIgpLCFgRzWnhlpHbjxWYM76+3GOkNm6+1kIHgXGBJ3PDj6Xrwm4HVVDQBbReSfRALDKgfL5Updmn3cnvZWNJ13wafHU2+Hg802S9mYHORkIFgFDBeRCiIB4BtA2xFBTwPfBB4UkQHACGCLg2VyrY5mH3cpGEDr3EH55MjP2GJ2SSakAYz8TB+bpWxMljkWCFQ1KCKzgBeJ9P8/oKobRORWoE5VF0U/O1dENgIh4Eeq2uxUmdyuvdnHXQ4E0Hq4KXy6mF2SCWmxpHLV0FJb9tqYLEopEIjId4EHgf3AXGA8cKOqLm7vOlV9Hni+zXu3xL1W4AfRP6abONZNFC/FCWlcvgiGRIJBKsteW1AwJvNSbRFcqap3i8h5QClwGfAI0G4gMLnJ8W6imDQmpCVrHcTPQbCJacY4I9VAINGfXwAeiXbxSHsXmNzmeDdRWylMSOsooWwT04xxRqqBoF5EFgMVwE0i0gcIO1cs0126pZsoXpornMKnCWWbmGaMM1INBFcBlcAWVf1ERI4Bvu1csUx36bZuonhprHAa32UEiUcZJeoyMsakLtVAcDrQoKofi8ilwAQii8mZAtDt3UQxqSaUo11GqeQRYl1G/g/99KnYY60DY1KQaiD4DTBORMYB1xMZOfQw8DmnCma6X7d3E0HHCeU2XUaJRhkl6jIKBMM817jCuoyMSUGqgSCoqioi04Bfq+rvReQqJwtmul9WuoniZbDLSLGhp8akKtVAsF9EbiIybHSyiHgAn3PFMtmStW6ieBnoMvIHwng8NvTUmFSkGghmEFke4kpVfV9Ejgf+27limWzLSjdRvC52GT2+ZBUTTh5lQ0+NSUFKgSD6y/9R4FQR+RKwUlUfdrZoJpuy3k0UrxNdRvtPKKbmtONTGnoaG2VkgcG4VapLTHydSAuglsjksntF5EequsDBspksS9ZN9Ozbz7bsidztASHFLqPjt9XB9l4dDj2NtQ7+tLrJ9lc2rpVq19DNwKmxHcRE5FhgCWCBwAXiu4m84uXpzU8TDAez2zqAdruMKsJhmLegpcsIoGpo6RFDT31FHgSS7q9sQcG4QaqBwNNmG8lmwONAeUwOiu8meu/Aeyz858LsJJHbaqfLSDiyyyg+IMSGnk4a1h/A1jYyrpZqIPiLiLwIPB49nkGbVUVNYYt1EzXsauDZt5/NXhI5kQRdRuHgYTwpjDKKsbWNjJulmiz+kYh8jcjewgBzVPUp54plclVOJZHjtekyanzlYYb19yUfZRQXFGKjjdrmEmxtI+MWKW9Mo6oLgYUOlsXkiZxLIrc1ZCLvDP2EYSf0SjzKKMnQ05hEQcH2SDCFrN1AICL7AU30EZF9ZY52pFQmL+RUEjmRZKOM2g49bVwWOb9NLgESJ5jbyyNYUDD5qN1AoKp9uqsgJv/kbBI5XqJRRvFBwVscOZ53wafHbbqNgKRrG1lQMIXAyc3rjQskSyJXl1XTsKshN7qKYhIFhfLJkZ8hf4e5hERzEiwomEJggcBkRHzroLqsGoCZi2fmRiI5kfigAJFf+h3lEiwomAJlgcBkTKx1AORuIjmRVHIJ7SSYOxpxZHMTTK6zQGAckfOJ5LY6yiW0s7ZRRyOObG6CyXUWCIwj2ksk53TrADoOCu1MVEsnKNjcBJMrLBAYxyRKJOdF6yBeusthpxEUbG6CyRUWCIzj8rp1EK+j5bDTDAqpzk2wZbKN0ywQmG5REK2DmM4klzs5N8GWyTbdwQKB6VYF1zqAjpPLXRiGastkm+5ggcB0u4JqHYCjQSGdZbJv+dIYVr/tp0/FHgsKJi2OBgIRmQrcDXiBuap6e5LzvkZkk5tTVbXOyTKZ3FEwrYN4XQkKSeYmQMfLZMeCQiisPNe4wloKJi2OBQIR8QL3AZ8HmoBVIrJIVTe2Oa8P8F3gdafKYnJXwbUO4mVobgKkPj9Bab/7CCzpbI7kZItgIrBZVbcAiMh8YBqwsc15/wX8AviRg2UxOa4gWwfxMjQ3AdoPCv5AGI8ncfdRkUdAxOYtmCOIaqJVpjNwY5ELgamqenX0+DLgNFWdFXfOBOBmVf2aiNQCP0zUNSQi1wDXAJSVlVXNnz+/U2U6cOAAvXv37tS1+Szf6r318Fbu3XkvIQ0h0f9ChCiSIq4ru46KHhUd3iNf6nz0vk3027ueHod289kdixHChBEQD6JK2FPEmnH/BUC/vevZ2+9kPuo7KuG9Nu8Jseb9g/Q/qgePbfITDEcaHWGNrCUvfLqmvACe6Gc+D1w8qpgDAWXUMV5OLPV2R9UzJl+edaalW+8pU6bUq2p1os+yliwWEQ/wS+CKjs5V1TnAHIDq6mqtqanp1HfW1tbS2WvzWb7Vu4Yaxu8a36p1oChhwjQd3USod6jD1kH+1Lkm8mP7Spj3KoT8eGJdRoTxaogJ3k3QML/dZbJjd4rV+8vb9iScyYwIoVDrrqRgGP6wKRDtSgrlXUshf551ZmWy3k4GgneBIXHHgxvvRfcAABCYSURBVKPvxfQBTgZqRQTgM8AiEbnAEsamoHMHiSSbm+AtBiSlZbI7mskcnyOwlVJNPCcDwSpguIhUEAkA3wAujn2oqvuAAbHj9rqGjHsVfO4gXrL9EiDlmczHb6uD7b2SBoXYMdjy2eZTjgUCVQ2KyCzgRSLDRx9Q1Q0icitQp6qLnPpuU1hc1zqAI/dLSHEmc0U4DPMWtJtsjunK8tkWFAqLozkCVX0eeL7Ne7ckObfGybKY/Oeq1kFbKQ5FFVJf8yieBQV3s5nFJq+k2jooaO0EhXDwMB6PJ7U1j6DLeyp0tEBe/GsLErnLAoHJSx21Dg7uO0i/Xf0Ks3UQr01QaHzlYYaNqe54JrPHC0iX91Rob4E8m7eQPywQmLzVXusgEA7w0uKXCi930J4hE3ln6CcMq65JYSZzOHqRdmkkUnsL5AVCCnQ829mCQvZZIDB5L1HrQFF35A6SSWUmc6xF0IU9FSD5AnmJ5i1YjiE3WSAwBaFt68Af8h+RO7jh1BvY599nQSF+WGoX91SA5AvkdXXeQuxaCxDOs0BgCkqsdfDk8icpGVjSkjvwh/zc9vpthDVcmMNNU9V2WGoGls9uby2k2DGkl3i2/EL3skBgCk7lwEr29t1LvxP6teQOBCGsYfcMN01XhvZUSDY8FdJMPFt+oVtZIDAFKz530Le4L3esusMdk9G6qqt7KsCR3U9pJp7TyS/ENuMB60rqLAsEpqDFcgcAw0uHJxxuWrczsqqJtRAS6MyeCrEF8rowRDWd/EIorCzasty6krrAAoFxjUTDTX0eH32L+zJz8Uz8Ib+1ENrTUVBou0BeF4eoxo6h48140ulKqo+uzGpB4lMWCIzrxHcZVZdVU7ezDn/Ib/mDdKSyQF4Xh6jG62gzniJv6l1Jtz63AX/QWg7xLBAYV4rvMgIo9hYnzB+4dshpOpItkNfVIaqxazvoSnp8ySq+ec6pQPtdSYFgmBfW72iZ8GZDVz9lgcC4XrLlKmzIaSdlYohqGvmF/ScUt/yy7mjm8/knD2JV44c2dLUNCwTGkDh/YENOMyytxHP6+YV4yRLQVUNLMzZ0FQqn1WCBwJg4qQ45tS6jLsrgEhgtm/FAh11JbY87O3S10FoNFgiMaaOjIafWZZRhXVwCoyIchoeeIJWupLY6O3S10FoNFgiMaYd1GXWzTuQXhDCEAtELUuhK2r4y5ZYDuKPVYIHAmBTYLOUs62gzHm8RKQ9VjZ/30IV8AxROq8ECgTEpSmWWsrUOukGizXjO+lbkuKOZz28+8+mEty4OXY0dQ+ZaDdna4c0CgTGdkMqWmZZQ7gaxzXhiv6w7mvl80jTYtjwjQ1fjZaLVkM0d3iwQGNMFNgchRyVLQA+ZmLmhq5DRVkM6O7zFWg6ZYoHAmC6yhHKOS5SA7urQ1fZaDZDyiqttu3+g4x3eAsEwK7Y0M0Yy91dkgcCYDLE5CHmos0NXk7UaOrHiauw4pqMd3nxFHiYN68/+rU0Z+2uwQGBMBtkchDyWztDVZK2GLnYrQcddS7EcQe3WzFXdAoExDrEuowKRTquhq91KacxzyCQLBMY4zLqMCkhHrYaudCt1cp5DJlggMKYbWJdRgctIt1Ia8xwyHBQsEBjTzazLyEXS6VZKdZ5D/N7QGeJoIBCRqcDdgBeYq6q3t/n8B8DVQBDYDVypqtucLJMxuSKdZSvA9lTOe6l0K6U0z8Ef+ZyqjBXNsUAgIl7gPuDzQBOwSkQWqerGuNPeAKpV9RMR+XfgDmCGU2UyJtekumzForcXteypbLmEApPuPAdvcSRovP1JxorgZItgIrBZVbcAiMh8YBrQEghUdWnc+SuASx0sjzE5LVGXkc/jQ9GWPZUtl+Ay7c2Qfrs2Y18jqpqxm7W6sciFwFRVvTp6fBlwmqrOSnL+r4H3VfVnCT67BrgGoKysrGr+/PmdKtOBAwfo3bt3p67NZ26sd77Xeevhrbx16C2GlwwH4N6d9xLSEBBZdEBRPHg4vffpHFN0DMNLhlPRoyLv690ZbqwzpF/vKVOm1KtqdaLPciJZLCKXAtXA5xJ9rqpzgDkA1dXVWlNT06nvqa2tpbPX5jM31jvf61xDTavj8bvGJ8wlrPpkVavhp29seoOvj/66q1oJ+f6sOyuT9XYyELwLDIk7Hhx9rxUROQe4Gficqh52sDzG5K1Uh5+GNMRLi1+yLiOTFo+D914FDBeRChEpBr4BtBrzJCLjgd8BF6jqLgfLYkzBqBxYydVjr+aCEy6g2FuMV7x4xENYwyjakmCeu24uDbsasl1ckwccaxGoalBEZgEvEhk++oCqbhCRW4E6VV0E/DfQG/ijiAC8o6oXOFUmYwpJouGn/pDfZiybtDmaI1DV54Hn27x3S9zrc5z8fmMKXdsuoyeXP0nJwJKkM5YtKJhEciJZbIzpusqBleztu5d+J/RLOGPZgoJJxgKBMQUm2Yzl9oKCJZfdzQKBMQUo0SijZEHB1jYyFgiMKXAdBQVLLhsLBMa4SLrLYVtQcAcLBMa4VEfLYVtQcA8LBMa4nCWXjQUCY4wll13OAoExphVLLruPBQJjTFKd2WsZbDe1fGOBwBiTklT3Wrbd1PKPBQJjTFqSJZc72k3NgkLuskBgjElboi6j6rLI5lc2FDX/WCAwxnRJfFAA0h6KakEh+ywQGGMyKp2hqBYUcoMFAmOMYywo5AcLBMaYbmFBIXdZIDDGdDsLCrnFAoExJqu6GhTe2PcG/Xb1A2wiW2dZIDDG5IzOBIWQhnjxxRcBbNmLTrJAYIzJSakGBUUJhAMALRParCspPRYIjDE5r72g4A/5KfJEfpWFNNTh8tlAqwlw1p1kgcAYk2faBoUnlz/J10//OkBKy2fH1kKKBQ/rTrJAYIzJY5UDK9nbd2/LL+72upLaroWUancSFH6rwQKBMabgpLIWkle8QPvdSe21GqBwAoQFAmNMQUu2FlL8L/P2upPgyFZDoQUICwTGGFdpGxg62o0Njmw1FFqAsEBgjDG0353UnQGiYVdDtwcMRwOBiEwF7ga8wFxVvb3N5z2Ah4EqoBmYoaqNTpbJGGM60lGrwckAERsS250tCscCgYh4gfuAzwNNwCoRWaSqG+NOuwrYo6onisg3gF8AM5wqkzHGdIWTASIQDrDknSUJd3hrGzBi8yEyxckWwURgs6puARCR+cA0ID4QTANmR18vAH4tIqKq6mC5jDEmozIRIHweH+ccfw6rd65uN2kdCAeo21nHiZyYsfKLU79zReRCYKqqXh09vgw4TVVnxZ2zPnpOU/T47eg5H7S51zXANQBlZWVV8+fP71SZDhw4QO/evTt1bT5zY73dWGdwZ73zsc5bD2/lrUNvMbxkOEDL64oeFS2fHeU5ioV7Fra0IgQhTBiveLmu7DqODRybVr2nTJlSr6rViT7Li2Sxqs4B5gBUV1drTU1Np+5TW1tLZ6/NZ26stxvrDO6sdz7WuYaalD774q4vJl0OI5P1djIQvAsMiTseHH0v0TlNIlIE9CWSNDbGGNdL1uWUaR5H7hqxChguIhUiUgx8A1jU5pxFwOXR1xcCr1h+wBhjupdjLQJVDYrILOBFIsNHH1DVDSJyK1CnqouA3wOPiMhm4EMiwcIYY0w3cjRHoKrPA8+3ee+WuNeHgIucLIMxxpj2Odk1ZIwxJg9YIDDGGJezQGCMMS7n2IQyp4jIbmBbJy8fAHzQ4VmFx431dmOdwZ31dmOdIf16D1XVYxN9kHeBoCtEpC7ZzLpC5sZ6u7HO4M56u7HOkNl6W9eQMca4nAUCY4xxObcFgjnZLkCWuLHebqwzuLPebqwzZLDersoRGGOMOZLbWgTGGGPasEBgjDEu55pAICJTReQfIrJZRG7MdnmcICJDRGSpiGwUkQ0i8t3o+8eIyEsi8lb0Z2m2y5ppIuIVkTdE5LnocYWIvB593k9EV8AtKCLST0QWiMgmEXlTRE53ybP+fvTf93oReVxESgrteYvIAyKyK7p5V+y9hM9WIu6J1n2tiExI9/tcEQji9k8+HxgNfFNERme3VI4IAter6mhgEnBttJ43Ai+r6nDg5ehxofku8Gbc8S+A/1HVE4E9RPbHLjR3A39R1VHAOCL1L+hnLSLHAd8BqlX1ZCIrG8f2Oy+k5/0QMLXNe8me7fnA8Oifa4DfpPtlrggExO2frKp+ILZ/ckFR1R2qujr6ej+RXwzHEanrvOhp84Dp2SmhM0RkMPBFYG70WICziOyDDYVZ577AvxJZyh1V9avqXgr8WUcVAT2jm1n1AnZQYM9bVV8jsjR/vGTPdhrwsEasAPqJyKB0vs8tgeA4YHvccVP0vYIlIuXAeOB1oExVd0Q/eh8oy1KxnPIr4AYgHD3uD+xV1WD0uBCfdwWwG3gw2iU2V0SOosCftaq+C9wJvEMkAOwD6in85w3Jn22Xf7+5JRC4ioj0BhYC31PVj+I/i+4AVzBjhkXkS8AuVa3Pdlm6WREwAfiNqo4HPqZNN1ChPWuAaL/4NCKB8LPAURzZhVLwMv1s3RIIUtk/uSCIiI9IEHhUVf8UfXtnrKkY/bkrW+VzwBnABSLSSKTL7ywifef9ol0HUJjPuwloUtXXo8cLiASGQn7WAOcAW1V1t6oGgD8R+TdQ6M8bkj/bLv9+c0sgSGX/5LwX7Rv/PfCmqv4y7qP4vaEvB57p7rI5RVVvUtXBqlpO5Lm+oqqXAEuJ7IMNBVZnAFV9H9guIiOjb50NbKSAn3XUO8AkEekV/fceq3dBP++oZM92EfCt6OihScC+uC6k1KiqK/4AXwD+CbwN3Jzt8jhUxzOJNBfXAg3RP18g0mf+MvAWsAQ4Jttldaj+NcBz0dfDgJXAZuCPQI9sl8+B+lYCddHn/TRQ6oZnDfwU2ASsBx4BehTa8wYeJ5IDCRBp/V2V7NkCQmRU5NvAOiIjqtL6PltiwhhjXM4tXUPGGGOSsEBgjDEuZ4HAGGNczgKBMca4nAUCY4xxOQsExjhMRGpiq6Iak4ssEBhjjMtZIDAmSkQuFZGVItIgIr+L7nFwQET+J7r+/csicmz03EoRWRFd//2puLXhTxSRJSKyRkRWi8gJ0dv3jts74NHorFhE5Pbo/hFrReTOLFXduJwFAmMAETkJmAGcoaqVQAi4hMiiZnWqOgZ4FfhJ9JKHgR+r6ilEZnPG3n8UuE9VxwH/QmR2KERWgv0ekf0whgFniEh/4CvAmOh9fuZsLY1JzAKBMRFnA1XAKhFpiB4PI7K09RPRc/4AnBndC6Cfqr4afX8e8K8i0gc4TlWfAlDVQ6r6SfSclarapKphIkt/lBNZQvkQ8HsR+SoQO9eYbmWBwJgIAeapamX0z0hVnZ3gvM6uyXI47nUIKNLI+vkTiawc+iXgL528tzFdYoHAmIiXgQtFZCC07A87lMj/I7FVLS8G/qqq+4A9IjI5+v5lwKsa2RWuSUSmR+/RQ0R6JfvC6L4RfVX1eeD7RLabNKbbFXV8ijGFT1U3ish/AotFxENk1cdriWz4MjH62S4ieQSILAP82+gv+i3At6PvXwb8TkRujd7jona+tg/wjIiUEGmR/CDD1TImJbb6qDHtEJEDqto72+UwxknWNWSMMS5nLQJjjHE5axEYY4zLWSAwxhiXs0BgjDEuZ4HAGGNczgKBMca43P8HwucVFEl2yYIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "v=vanilla(X,y)\n",
    "m=momentum(X,y)\n",
    "a=adam(X,y)\n",
    "plt.plot(v,'.',label='vanilla')\n",
    "plt.plot(m,'.',label='momentum')\n",
    "plt.plot(a,'.',label='adam')\n",
    "plt.legend()\n",
    "plt.xlabel('epochs')\n",
    "plt.ylabel('loss')\n",
    "plt.grid()"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "D54eDEv6jkO4"
   ],
   "name": "srikar_Copy_of_Backpropagation_Assignment.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}