word2vec_skipgram.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.511310Z",
     "start_time": "2019-07-14T06:37:40.375961Z"
    }
   },
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Prep"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.534268Z",
     "start_time": "2019-07-14T06:37:40.513450Z"
    },
    "code_folding": [
     5
    ]
   },
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    # obtains tokens with a least 1 alphabet\n",
    "    pattern = re.compile(r'[A-Za-z]+[\\w^\\']*|[\\w^\\']*[A-Za-z]+[\\w^\\']*')\n",
    "    return pattern.findall(text.lower())\n",
    "\n",
    "def mapping(tokens):\n",
    "    word_to_id = dict()\n",
    "    id_to_word = dict()\n",
    "\n",
    "    for i, token in enumerate(set(tokens)):\n",
    "        word_to_id[token] = i\n",
    "        id_to_word[i] = token\n",
    "\n",
    "    return word_to_id, id_to_word\n",
    "\n",
    "def generate_training_data(tokens, word_to_id, window_size):\n",
    "    N = len(tokens)\n",
    "    X, Y = [], []\n",
    "\n",
    "    for i in range(N):\n",
    "        nbr_inds = list(range(max(0, i - window_size), i)) + \\\n",
    "                   list(range(i + 1, min(N, i + window_size + 1)))\n",
    "        print(\"At index \", i, \"we have: \", nbr_inds)\n",
    "        for j in nbr_inds:\n",
    "            X.append(word_to_id[tokens[i]])\n",
    "            Y.append(word_to_id[tokens[j]])\n",
    "            \n",
    "    X = np.array(X)\n",
    "    X = np.expand_dims(X, axis=0)\n",
    "    Y = np.array(Y)\n",
    "    Y = np.expand_dims(Y, axis=0)\n",
    "            \n",
    "    return X, Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.553497Z",
     "start_time": "2019-07-14T06:37:40.535947Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['after', 'the', 'deduction', 'of', 'the', 'costs', 'of', 'investing', 'beating', 'the', 'stock', 'market', 'is', 'a', \"loser's\", 'game']\n"
     ]
    }
   ],
   "source": [
    "doc = \"After the deduction of the costs of investing, \" \\\n",
    "      \"beating the stock market is a loser's game.\"\n",
    "tokens = tokenize(doc)\n",
    "print(tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.575126Z",
     "start_time": "2019-07-14T06:37:40.555535Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'costs': 0, 'beating': 1, 'game': 2, 'a': 3, 'the': 4, 'after': 5, 'of': 6, 'stock': 7, 'investing': 8, 'is': 9, 'deduction': 10, \"loser's\": 11, 'market': 12}\n",
      "{0: 'costs', 1: 'beating', 2: 'game', 3: 'a', 4: 'the', 5: 'after', 6: 'of', 7: 'stock', 8: 'investing', 9: 'is', 10: 'deduction', 11: \"loser's\", 12: 'market'}\n"
     ]
    }
   ],
   "source": [
    "word_to_id, id_to_word = mapping(tokens)\n",
    "print(word_to_id)\n",
    "print(id_to_word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.607739Z",
     "start_time": "2019-07-14T06:37:40.578574Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "At index  0 we have:  [1, 2, 3]\n",
      "At index  1 we have:  [0, 2, 3, 4]\n",
      "At index  2 we have:  [0, 1, 3, 4, 5]\n",
      "At index  3 we have:  [0, 1, 2, 4, 5, 6]\n",
      "At index  4 we have:  [1, 2, 3, 5, 6, 7]\n",
      "At index  5 we have:  [2, 3, 4, 6, 7, 8]\n",
      "At index  6 we have:  [3, 4, 5, 7, 8, 9]\n",
      "At index  7 we have:  [4, 5, 6, 8, 9, 10]\n",
      "At index  8 we have:  [5, 6, 7, 9, 10, 11]\n",
      "At index  9 we have:  [6, 7, 8, 10, 11, 12]\n",
      "At index  10 we have:  [7, 8, 9, 11, 12, 13]\n",
      "At index  11 we have:  [8, 9, 10, 12, 13, 14]\n",
      "At index  12 we have:  [9, 10, 11, 13, 14, 15]\n",
      "At index  13 we have:  [10, 11, 12, 14, 15]\n",
      "At index  14 we have:  [11, 12, 13, 15]\n",
      "At index  15 we have:  [12, 13, 14]\n",
      "[[ 5  5  5  4  4  4  4 10 10 10 10 10  6  6  6  6  6  6  4  4  4  4  4  4\n",
      "   0  0  0  0  0  0  6  6  6  6  6  6  8  8  8  8  8  8  1  1  1  1  1  1\n",
      "   4  4  4  4  4  4  7  7  7  7  7  7 12 12 12 12 12 12  9  9  9  9  9  9\n",
      "   3  3  3  3  3 11 11 11 11  2  2  2]]\n",
      "[[ 4 10  6  5 10  6  4  5  4  6  4  0  5  4 10  4  0  6  4 10  6  0  6  8\n",
      "  10  6  4  6  8  1  6  4  0  8  1  4  4  0  6  1  4  7  0  6  8  4  7 12\n",
      "   6  8  1  7 12  9  8  1  4 12  9  3  1  4  7  9  3 11  4  7 12  3 11  2\n",
      "   7 12  9 11  2 12  9  3  2  9  3 11]]\n"
     ]
    }
   ],
   "source": [
    "X, Y = generate_training_data(tokens, word_to_id, 3)\n",
    "print(X)\n",
    "print(Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.638952Z",
     "start_time": "2019-07-14T06:37:40.609758Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "84\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n",
      "(13, 84)\n",
      "\n",
      "[ 4 10  6  5 10  6  4  5  4  6  4  0  5  4 10  4  0  6  4 10  6  0  6  8\n",
      " 10  6  4  6  8  1  6  4  0  8  1  4  4  0  6  1  4  7  0  6  8  4  7 12\n",
      "  6  8  1  7 12  9  8  1  4 12  9  3  1  4  7  9  3 11  4  7 12  3 11  2\n",
      "  7 12  9 11  2 12  9  3  2  9  3 11]\n",
      "[[0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 0.]\n",
      " ...\n",
      " [0. 1. 0. ... 0. 0. 0.]\n",
      " [0. 0. 0. ... 0. 0. 1.]\n",
      " [0. 0. 0. ... 0. 0. 0.]]\n"
     ]
    }
   ],
   "source": [
    "vocab_size = len(id_to_word)\n",
    "m = Y.shape[1]\n",
    "print(m)\n",
    "# turn Y into one hot encoding\n",
    "Y_one_hot = np.zeros((vocab_size, m))\n",
    "print(Y_one_hot)\n",
    "print(Y_one_hot.shape)\n",
    "print()\n",
    "print(Y.flatten())\n",
    "Y_one_hot[Y.flatten(), np.arange(m)] = 1\n",
    "print(Y_one_hot)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.687344Z",
     "start_time": "2019-07-14T06:37:40.644010Z"
    }
   },
   "outputs": [],
   "source": [
    "def initialize_wrd_emb(vocab_size, emb_size):\n",
    "    \"\"\"\n",
    "    vocab_size: int. vocabulary size of your corpus or training data\n",
    "    emb_size: int. word embedding size. How many dimensions to represent each vocabulary\n",
    "    \"\"\"\n",
    "    WRD_EMB = np.random.randn(vocab_size, emb_size) * 0.01\n",
    "    \n",
    "    assert(WRD_EMB.shape == (vocab_size, emb_size))\n",
    "    return WRD_EMB\n",
    "\n",
    "def initialize_dense(input_size, output_size):\n",
    "    \"\"\"\n",
    "    input_size: int. size of the input to the dense layer\n",
    "    output_szie: int. size of the output out of the dense layer\n",
    "    \"\"\"\n",
    "    W = np.random.randn(output_size, input_size) * 0.01\n",
    "    \n",
    "    assert(W.shape == (output_size, input_size))\n",
    "    return W\n",
    "\n",
    "def initialize_parameters(vocab_size, emb_size):\n",
    "    WRD_EMB = initialize_wrd_emb(vocab_size, emb_size)\n",
    "    W = initialize_dense(emb_size, vocab_size)\n",
    "    \n",
    "    parameters = {}\n",
    "    parameters['WRD_EMB'] = WRD_EMB\n",
    "    parameters['W'] = W\n",
    "    \n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Forward Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.719881Z",
     "start_time": "2019-07-14T06:37:40.689970Z"
    }
   },
   "outputs": [],
   "source": [
    "def ind_to_word_vecs(inds, parameters):\n",
    "    \"\"\"\n",
    "    inds: numpy array. shape: (1, m)\n",
    "    parameters: dict. weights to be trained\n",
    "    \"\"\"\n",
    "    m = inds.shape[1]\n",
    "    WRD_EMB = parameters['WRD_EMB']\n",
    "    word_vec = WRD_EMB[inds.flatten(), :].T\n",
    "    \n",
    "    assert(word_vec.shape == (WRD_EMB.shape[1], m))\n",
    "    \n",
    "    return word_vec\n",
    "\n",
    "def linear_dense(word_vec, parameters):\n",
    "    \"\"\"\n",
    "    word_vec: numpy array. shape: (emb_size, m)\n",
    "    parameters: dict. weights to be trained\n",
    "    \"\"\"\n",
    "    m = word_vec.shape[1]\n",
    "    W = parameters['W']\n",
    "    Z = np.dot(W, word_vec)\n",
    "    \n",
    "    assert(Z.shape == (W.shape[0], m))\n",
    "    \n",
    "    return W, Z\n",
    "\n",
    "def softmax(Z):\n",
    "    \"\"\"\n",
    "    Z: output out of the dense layer. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    softmax_out = np.divide(np.exp(Z), np.sum(np.exp(Z), axis=0, keepdims=True) + 0.001)\n",
    "    \n",
    "    assert(softmax_out.shape == Z.shape)\n",
    "\n",
    "    return softmax_out\n",
    "\n",
    "def forward_propagation(inds, parameters):\n",
    "    word_vec = ind_to_word_vecs(inds, parameters)\n",
    "    W, Z = linear_dense(word_vec, parameters)\n",
    "    softmax_out = softmax(Z)\n",
    "    \n",
    "    caches = {}\n",
    "    caches['inds'] = inds\n",
    "    caches['word_vec'] = word_vec\n",
    "    caches['W'] = W\n",
    "    caches['Z'] = Z\n",
    "    \n",
    "    return softmax_out, caches"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cost Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.743541Z",
     "start_time": "2019-07-14T06:37:40.723714Z"
    }
   },
   "outputs": [],
   "source": [
    "def cross_entropy(softmax_out, Y):\n",
    "    \"\"\"\n",
    "    softmax_out: output out of softmax. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    m = softmax_out.shape[1]\n",
    "    cost = -(1 / m) * np.sum(np.sum(Y * np.log(softmax_out + 0.001), axis=0, keepdims=True), axis=1)\n",
    "    return cost"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Backward Propagation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:40.776175Z",
     "start_time": "2019-07-14T06:37:40.747374Z"
    }
   },
   "outputs": [],
   "source": [
    "def softmax_backward(Y, softmax_out):\n",
    "    \"\"\"\n",
    "    Y: labels of training data. shape: (vocab_size, m)\n",
    "    softmax_out: output out of softmax. shape: (vocab_size, m)\n",
    "    \"\"\"\n",
    "    dL_dZ = softmax_out - Y\n",
    "    \n",
    "    assert(dL_dZ.shape == softmax_out.shape)\n",
    "    return dL_dZ\n",
    "\n",
    "def dense_backward(dL_dZ, caches):\n",
    "    \"\"\"\n",
    "    dL_dZ: shape: (vocab_size, m)\n",
    "    caches: dict. results from each steps of forward propagation\n",
    "    \"\"\"\n",
    "    W = caches['W']\n",
    "    word_vec = caches['word_vec']\n",
    "    m = word_vec.shape[1]\n",
    "    \n",
    "    dL_dW = (1 / m) * np.dot(dL_dZ, word_vec.T)\n",
    "    dL_dword_vec = np.dot(W.T, dL_dZ)\n",
    "\n",
    "    assert(W.shape == dL_dW.shape)\n",
    "    assert(word_vec.shape == dL_dword_vec.shape)\n",
    "    \n",
    "    return dL_dW, dL_dword_vec\n",
    "\n",
    "def backward_propagation(Y, softmax_out, caches):\n",
    "    dL_dZ = softmax_backward(Y, softmax_out)\n",
    "    dL_dW, dL_dword_vec = dense_backward(dL_dZ, caches)\n",
    "    \n",
    "    gradients = dict()\n",
    "    gradients['dL_dZ'] = dL_dZ\n",
    "    gradients['dL_dW'] = dL_dW\n",
    "    gradients['dL_dword_vec'] = dL_dword_vec\n",
    "    \n",
    "    return gradients\n",
    "\n",
    "def update_parameters(parameters, caches, gradients, learning_rate):\n",
    "    vocab_size, emb_size = parameters['WRD_EMB'].shape\n",
    "    inds = caches['inds']\n",
    "    dL_dword_vec = gradients['dL_dword_vec']\n",
    "    m = inds.shape[-1]\n",
    "    \n",
    "    parameters['WRD_EMB'][inds.flatten(), :] -= dL_dword_vec.T * learning_rate\n",
    "\n",
    "    parameters['W'] -= learning_rate * gradients['dL_dW']\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:41.143651Z",
     "start_time": "2019-07-14T06:37:40.778463Z"
    }
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "def skipgram_model_training(X, Y, vocab_size, emb_size, learning_rate, epochs, batch_size=256, parameters=None, print_cost=False, plot_cost=True):\n",
    "    costs = []\n",
    "    m = X.shape[1]\n",
    "    \n",
    "    if parameters is None:\n",
    "        parameters = initialize_parameters(vocab_size, emb_size)\n",
    "    \n",
    "    begin_time = datetime.now()\n",
    "    for epoch in range(epochs):\n",
    "        epoch_cost = 0\n",
    "        batch_inds = list(range(0, m, batch_size))\n",
    "        np.random.shuffle(batch_inds)\n",
    "        for i in batch_inds:\n",
    "            X_batch = X[:, i:i+batch_size]\n",
    "            Y_batch = Y[:, i:i+batch_size]\n",
    "\n",
    "            softmax_out, caches = forward_propagation(X_batch, parameters)\n",
    "            #print(\"softmax size is: \", softmax_out.shape)\n",
    "            gradients = backward_propagation(Y_batch, softmax_out, caches)\n",
    "            update_parameters(parameters, caches, gradients, learning_rate)\n",
    "            cost = cross_entropy(softmax_out, Y_batch)\n",
    "            epoch_cost += np.squeeze(cost) # turn ndarray into floating number\n",
    "            \n",
    "        costs.append(epoch_cost)\n",
    "        if print_cost and epoch % 10 == 0:\n",
    "            print(\"Cost after epoch {}: {}\".format(epoch, epoch_cost))\n",
    "        if epoch % 100 == 0:\n",
    "            learning_rate *= 0.98\n",
    "    end_time = datetime.now()\n",
    "    print('training time: {}'.format(end_time - begin_time))\n",
    "            \n",
    "    if plot_cost:\n",
    "        plt.plot(np.arange(epochs), costs)\n",
    "        plt.xlabel('# of epochs')\n",
    "        plt.ylabel('cost')\n",
    "    return parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.129201Z",
     "start_time": "2019-07-14T06:37:41.145759Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after epoch 0: 2.552108809331128\n",
      "Cost after epoch 10: 2.5518043033291877\n",
      "Cost after epoch 20: 2.5514883314170094\n",
      "Cost after epoch 30: 2.551144709162928\n",
      "Cost after epoch 40: 2.5507571123836366\n",
      "Cost after epoch 50: 2.550307986453164\n",
      "Cost after epoch 60: 2.549778045210645\n",
      "Cost after epoch 70: 2.5491457292667064\n",
      "Cost after epoch 80: 2.54838661324859\n",
      "Cost after epoch 90: 2.5474727537866486\n",
      "Cost after epoch 100: 2.546371973799601\n",
      "Cost after epoch 110: 2.545073063542792\n",
      "Cost after epoch 120: 2.5435207775474984\n",
      "Cost after epoch 130: 2.5416660959411246\n",
      "Cost after epoch 140: 2.539455493714363\n",
      "Cost after epoch 150: 2.5368278790049206\n",
      "Cost after epoch 160: 2.5337141377780945\n",
      "Cost after epoch 170: 2.5300370437191284\n",
      "Cost after epoch 180: 2.525711764961187\n",
      "Cost after epoch 190: 2.520647281454295\n",
      "Cost after epoch 200: 2.5147491148013263\n",
      "Cost after epoch 210: 2.508055026109384\n",
      "Cost after epoch 220: 2.5004022573582705\n",
      "Cost after epoch 230: 2.4917133166764502\n",
      "Cost after epoch 240: 2.4819502206056656\n",
      "Cost after epoch 250: 2.471114875102046\n",
      "Cost after epoch 260: 2.459262764352024\n",
      "Cost after epoch 270: 2.446514711117789\n",
      "Cost after epoch 280: 2.4330628439415793\n",
      "Cost after epoch 290: 2.419166203656568\n",
      "Cost after epoch 300: 2.405132295698569\n",
      "Cost after epoch 310: 2.391530197466395\n",
      "Cost after epoch 320: 2.378415046076064\n",
      "Cost after epoch 330: 2.3659768423401237\n",
      "Cost after epoch 340: 2.354346549491856\n",
      "Cost after epoch 350: 2.3435610246754606\n",
      "Cost after epoch 360: 2.33357370740946\n",
      "Cost after epoch 370: 2.3242764293558493\n",
      "Cost after epoch 380: 2.31552414975126\n",
      "Cost after epoch 390: 2.307156918189172\n",
      "Cost after epoch 400: 2.2990169710653805\n",
      "Cost after epoch 410: 2.291106356484169\n",
      "Cost after epoch 420: 2.283179403710382\n",
      "Cost after epoch 430: 2.275133603989085\n",
      "Cost after epoch 440: 2.2669116609158175\n",
      "Cost after epoch 450: 2.2584854405969166\n",
      "Cost after epoch 460: 2.2498540671016203\n",
      "Cost after epoch 470: 2.241040408795061\n",
      "Cost after epoch 480: 2.23208641512575\n",
      "Cost after epoch 490: 2.2230477485134545\n",
      "Cost after epoch 500: 2.2139881363360585\n",
      "Cost after epoch 510: 2.205135799732093\n",
      "Cost after epoch 520: 2.196405558201023\n",
      "Cost after epoch 530: 2.187833847187759\n",
      "Cost after epoch 540: 2.179467242366664\n",
      "Cost after epoch 550: 2.171343225700342\n",
      "Cost after epoch 560: 2.16349019887128\n",
      "Cost after epoch 570: 2.1559282831253657\n",
      "Cost after epoch 580: 2.148670526747497\n",
      "Cost after epoch 590: 2.141724191588356\n",
      "Cost after epoch 600: 2.135091909547528\n",
      "Cost after epoch 610: 2.128884124833081\n",
      "Cost after epoch 620: 2.1229861709869127\n",
      "Cost after epoch 630: 2.1173791337358527\n",
      "Cost after epoch 640: 2.1120553933885122\n",
      "Cost after epoch 650: 2.10700602550316\n",
      "Cost after epoch 660: 2.102221174885324\n",
      "Cost after epoch 670: 2.0976903960161524\n",
      "Cost after epoch 680: 2.0934029537732046\n",
      "Cost after epoch 690: 2.0893480773261928\n",
      "Cost after epoch 700: 2.0855151627155464\n",
      "Cost after epoch 710: 2.0819576720973343\n",
      "Cost after epoch 720: 2.0786016428961913\n",
      "Cost after epoch 730: 2.0754307315900693\n",
      "Cost after epoch 740: 2.072436553646163\n",
      "Cost after epoch 750: 2.0696111899664755\n",
      "Cost after epoch 760: 2.066947169249545\n",
      "Cost after epoch 770: 2.0644374394391876\n",
      "Cost after epoch 780: 2.062075333507712\n",
      "Cost after epoch 790: 2.0598545337639993\n",
      "Cost after epoch 800: 2.0577690378209854\n",
      "Cost after epoch 810: 2.0558475274502483\n",
      "Cost after epoch 820: 2.054049395744505\n",
      "Cost after epoch 830: 2.0523656449042136\n",
      "Cost after epoch 840: 2.0507915610414678\n",
      "Cost after epoch 850: 2.049322630551222\n",
      "Cost after epoch 860: 2.047954532222008\n",
      "Cost after epoch 870: 2.0466831309485856\n",
      "Cost after epoch 880: 2.0455044717171513\n",
      "Cost after epoch 890: 2.044414772649454\n",
      "Cost after epoch 900: 2.043410416138781\n",
      "Cost after epoch 910: 2.0425041067957372\n",
      "Cost after epoch 920: 2.041675253723047\n",
      "Cost after epoch 930: 2.0409188725262264\n",
      "Cost after epoch 940: 2.0402320340758204\n",
      "Cost after epoch 950: 2.0396118894625603\n",
      "Cost after epoch 960: 2.039055649972767\n",
      "Cost after epoch 970: 2.0385605680842027\n",
      "Cost after epoch 980: 2.0381239201814383\n",
      "Cost after epoch 990: 2.0377429916660437\n",
      "Cost after epoch 1000: 2.037415065033474\n",
      "Cost after epoch 1010: 2.0371422265340424\n",
      "Cost after epoch 1020: 2.0369157130237405\n",
      "Cost after epoch 1030: 2.0367322811629562\n",
      "Cost after epoch 1040: 2.0365893108379165\n",
      "Cost after epoch 1050: 2.036484175194644\n",
      "Cost after epoch 1060: 2.0364142457912227\n",
      "Cost after epoch 1070: 2.0363769004871837\n",
      "Cost after epoch 1080: 2.0363695331734153\n",
      "Cost after epoch 1090: 2.0363895646421146\n",
      "Cost after epoch 1100: 2.0364344540323276\n",
      "Cost after epoch 1110: 2.0365005380940118\n",
      "Cost after epoch 1120: 2.0365857182777094\n",
      "Cost after epoch 1130: 2.0366878514331614\n",
      "Cost after epoch 1140: 2.0368047883632983\n",
      "Cost after epoch 1150: 2.036934466597797\n",
      "Cost after epoch 1160: 2.0370749159128327\n",
      "Cost after epoch 1170: 2.0372242640534597\n",
      "Cost after epoch 1180: 2.037380742041862\n",
      "Cost after epoch 1190: 2.037542688826544\n",
      "Cost after epoch 1200: 2.037708555196526\n",
      "Cost after epoch 1210: 2.037874063805531\n",
      "Cost after epoch 1220: 2.0380403971256915\n",
      "Cost after epoch 1230: 2.038206728084717\n",
      "Cost after epoch 1240: 2.038372035745445\n",
      "Cost after epoch 1250: 2.0385354132076743\n",
      "Cost after epoch 1260: 2.0386960655388373\n",
      "Cost after epoch 1270: 2.0388533083207196\n",
      "Cost after epoch 1280: 2.03900656599933\n",
      "Cost after epoch 1290: 2.0391553697137583\n",
      "Cost after epoch 1300: 2.039299354461941\n",
      "Cost after epoch 1310: 2.0394359889225737\n",
      "Cost after epoch 1320: 2.039567310813668\n",
      "Cost after epoch 1330: 2.0396934983401493\n",
      "Cost after epoch 1340: 2.039814553808507\n",
      "Cost after epoch 1350: 2.039930554148657\n",
      "Cost after epoch 1360: 2.040041642438294\n",
      "Cost after epoch 1370: 2.0401480201451467\n",
      "Cost after epoch 1380: 2.040249939273818\n",
      "Cost after epoch 1390: 2.040347694246095\n",
      "Cost after epoch 1400: 2.0404416135629475\n",
      "Cost after epoch 1410: 2.0405306300161787\n",
      "Cost after epoch 1420: 2.040616480944341\n",
      "Cost after epoch 1430: 2.04069968106141\n",
      "Cost after epoch 1440: 2.0407805901238367\n",
      "Cost after epoch 1450: 2.0408595647793364\n",
      "Cost after epoch 1460: 2.0409369509558473\n",
      "Cost after epoch 1470: 2.0410130783080414\n",
      "Cost after epoch 1480: 2.0410882558124657\n",
      "Cost after epoch 1490: 2.041162768295832\n",
      "Cost after epoch 1500: 2.0412368738571676\n",
      "Cost after epoch 1510: 2.041309641986859\n",
      "Cost after epoch 1520: 2.041382294385311\n",
      "Cost after epoch 1530: 2.0414551135164842\n",
      "Cost after epoch 1540: 2.041528234697075\n",
      "Cost after epoch 1550: 2.0416017698015363\n",
      "Cost after epoch 1560: 2.0416758072146632\n",
      "Cost after epoch 1570: 2.0417504135810165\n",
      "Cost after epoch 1580: 2.041825636181555\n",
      "Cost after epoch 1590: 2.041901505572922\n",
      "Cost after epoch 1600: 2.0419780383377577\n",
      "Cost after epoch 1610: 2.042054008637997\n",
      "Cost after epoch 1620: 2.042130479527315\n",
      "Cost after epoch 1630: 2.0422075683025147\n",
      "Cost after epoch 1640: 2.0422852623878938\n",
      "Cost after epoch 1650: 2.04236354979637\n",
      "Cost after epoch 1660: 2.0424424189058636\n",
      "Cost after epoch 1670: 2.042521859430253\n",
      "Cost after epoch 1680: 2.042601863398967\n",
      "Cost after epoch 1690: 2.042682425845396\n",
      "Cost after epoch 1700: 2.0427635451318653\n",
      "Cost after epoch 1710: 2.042843907725764\n",
      "Cost after epoch 1720: 2.0429246644160886\n",
      "Cost after epoch 1730: 2.043005955054642\n",
      "Cost after epoch 1740: 2.043087785201725\n",
      "Cost after epoch 1750: 2.0431701625307452\n",
      "Cost after epoch 1760: 2.0432530932513946\n",
      "Cost after epoch 1770: 2.043336579951866\n",
      "Cost after epoch 1780: 2.0434206196664246\n",
      "Cost after epoch 1790: 2.0435052019169277\n",
      "Cost after epoch 1800: 2.043590306708909\n",
      "Cost after epoch 1810: 2.0436745139028134\n",
      "Cost after epoch 1820: 2.0437589960978175\n",
      "Cost after epoch 1830: 2.0438438377919885\n",
      "Cost after epoch 1840: 2.0439289648203864\n",
      "Cost after epoch 1850: 2.044014288512574\n",
      "Cost after epoch 1860: 2.044099702175191\n",
      "Cost after epoch 1870: 2.0441850794799334\n",
      "Cost after epoch 1880: 2.04427027349355\n",
      "Cost after epoch 1890: 2.0443551160582474\n",
      "Cost after epoch 1900: 2.044439417462275\n",
      "Cost after epoch 1910: 2.0445216230691\n",
      "Cost after epoch 1920: 2.0446027340115607\n",
      "Cost after epoch 1930: 2.044682650553075\n",
      "Cost after epoch 1940: 2.0447611142517887\n",
      "Cost after epoch 1950: 2.044837851937696\n",
      "Cost after epoch 1960: 2.044912574644982\n",
      "Cost after epoch 1970: 2.04498497848918\n",
      "Cost after epoch 1980: 2.0450547461873194\n",
      "Cost after epoch 1990: 2.045121548894064\n",
      "Cost after epoch 2000: 2.045185048256697\n",
      "Cost after epoch 2010: 2.045244004891737\n",
      "Cost after epoch 2020: 2.045299015755766\n",
      "Cost after epoch 2030: 2.0453498435978954\n",
      "Cost after epoch 2040: 2.045396154996128\n",
      "Cost after epoch 2050: 2.045437621684871\n",
      "Cost after epoch 2060: 2.0454739207937003\n",
      "Cost after epoch 2070: 2.0455047367508863\n",
      "Cost after epoch 2080: 2.04552976358232\n",
      "Cost after epoch 2090: 2.0455487072815814\n",
      "Cost after epoch 2100: 2.0455612881480296\n",
      "Cost after epoch 2110: 2.0455673421716716\n",
      "Cost after epoch 2120: 2.0455667931086152\n",
      "Cost after epoch 2130: 2.045559421468857\n",
      "Cost after epoch 2140: 2.045545036282931\n",
      "Cost after epoch 2150: 2.0455234712458505\n",
      "Cost after epoch 2160: 2.045494584053213\n",
      "Cost after epoch 2170: 2.0454582571038644\n",
      "Cost after epoch 2180: 2.0454143983929063\n",
      "Cost after epoch 2190: 2.0453629423118396\n",
      "Cost after epoch 2200: 2.0453038502784784\n",
      "Cost after epoch 2210: 2.0452385233923\n",
      "Cost after epoch 2220: 2.0451660006329972\n",
      "Cost after epoch 2230: 2.045086171907359\n",
      "Cost after epoch 2240: 2.0449991037554516\n",
      "Cost after epoch 2250: 2.0449048898233375\n",
      "Cost after epoch 2260: 2.044803648568038\n",
      "Cost after epoch 2270: 2.0446955222208767\n",
      "Cost after epoch 2280: 2.044580675941913\n",
      "Cost after epoch 2290: 2.0444592969281445\n",
      "Cost after epoch 2300: 2.0443315934228683\n",
      "Cost after epoch 2310: 2.044200394517039\n",
      "Cost after epoch 2320: 2.0440638401843203\n",
      "Cost after epoch 2330: 2.043921893723274\n",
      "Cost after epoch 2340: 2.0437748192558245\n",
      "Cost after epoch 2350: 2.0436228961690435\n",
      "Cost after epoch 2360: 2.0434664161888456\n",
      "Cost after epoch 2370: 2.0433056817039166\n",
      "Cost after epoch 2380: 2.043141004359267\n",
      "Cost after epoch 2390: 2.0429727037100123\n",
      "Cost after epoch 2400: 2.0428011058896165\n",
      "Cost after epoch 2410: 2.042629844531822\n",
      "Cost after epoch 2420: 2.0424564011267266\n",
      "Cost after epoch 2430: 2.042280725365383\n",
      "Cost after epoch 2440: 2.0421031378266723\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cost after epoch 2450: 2.0419239606006614\n",
      "Cost after epoch 2460: 2.0417435147743266\n",
      "Cost after epoch 2470: 2.0415621191202975\n",
      "Cost after epoch 2480: 2.0413800890720286\n",
      "Cost after epoch 2490: 2.041197735794402\n",
      "Cost after epoch 2500: 2.041015365305019\n",
      "Cost after epoch 2510: 2.0408366817242705\n",
      "Cost after epoch 2520: 2.0406589139875764\n",
      "Cost after epoch 2530: 2.040481945772729\n",
      "Cost after epoch 2540: 2.0403060400404236\n",
      "Cost after epoch 2550: 2.0401314526005048\n",
      "Cost after epoch 2560: 2.03995843023839\n",
      "Cost after epoch 2570: 2.039787209952189\n",
      "Cost after epoch 2580: 2.03961801843256\n",
      "Cost after epoch 2590: 2.039451071612932\n",
      "Cost after epoch 2600: 2.039286574250334\n",
      "Cost after epoch 2610: 2.0391277334411373\n",
      "Cost after epoch 2620: 2.03897193080323\n",
      "Cost after epoch 2630: 2.0388189793448563\n",
      "Cost after epoch 2640: 2.038669028281103\n",
      "Cost after epoch 2650: 2.0385222157870273\n",
      "Cost after epoch 2660: 2.0383786675395776\n",
      "Cost after epoch 2670: 2.038238496294052\n",
      "Cost after epoch 2680: 2.0381018016630357\n",
      "Cost after epoch 2690: 2.0379686699456374\n",
      "Cost after epoch 2700: 2.037839173977114\n",
      "Cost after epoch 2710: 2.0377157233592755\n",
      "Cost after epoch 2720: 2.03759612253524\n",
      "Cost after epoch 2730: 2.0374801273004626\n",
      "Cost after epoch 2740: 2.037367756665517\n",
      "Cost after epoch 2750: 2.037259017650091\n",
      "Cost after epoch 2760: 2.0371539041743505\n",
      "Cost after epoch 2770: 2.0370523969663257\n",
      "Cost after epoch 2780: 2.0369544636733647\n",
      "Cost after epoch 2790: 2.036860059041198\n",
      "Cost after epoch 2800: 2.036769125138182\n",
      "Cost after epoch 2810: 2.03668325047974\n",
      "Cost after epoch 2820: 2.0366007453418633\n",
      "Cost after epoch 2830: 2.0365213244565474\n",
      "Cost after epoch 2840: 2.0364448905938044\n",
      "Cost after epoch 2850: 2.0363713373155257\n",
      "Cost after epoch 2860: 2.0363005483945344\n",
      "Cost after epoch 2870: 2.036232398249151\n",
      "Cost after epoch 2880: 2.0361667525816327\n",
      "Cost after epoch 2890: 2.0361034690882764\n",
      "Cost after epoch 2900: 2.0360423982172606\n",
      "Cost after epoch 2910: 2.0359845360969544\n",
      "Cost after epoch 2920: 2.0359286201707874\n",
      "Cost after epoch 2930: 2.0358743559038763\n",
      "Cost after epoch 2940: 2.035821584706443\n",
      "Cost after epoch 2950: 2.0357701460716204\n",
      "Cost after epoch 2960: 2.035719877394215\n",
      "Cost after epoch 2970: 2.035670614761938\n",
      "Cost after epoch 2980: 2.035622193898434\n",
      "Cost after epoch 2990: 2.0355744511265614\n",
      "Cost after epoch 3000: 2.0355272243261875\n",
      "Cost after epoch 3010: 2.035481297289709\n",
      "Cost after epoch 3020: 2.0354356630156034\n",
      "Cost after epoch 3030: 2.0353900666765528\n",
      "Cost after epoch 3040: 2.03534437124062\n",
      "Cost after epoch 3050: 2.035298446002419\n",
      "Cost after epoch 3060: 2.03525216625499\n",
      "Cost after epoch 3070: 2.0352054138793294\n",
      "Cost after epoch 3080: 2.0351580780258183\n",
      "Cost after epoch 3090: 2.0351100557612973\n",
      "Cost after epoch 3100: 2.035061252660218\n",
      "Cost after epoch 3110: 2.0350125830429935\n",
      "Cost after epoch 3120: 2.034963112634546\n",
      "Cost after epoch 3130: 2.0349126631700205\n",
      "Cost after epoch 3140: 2.0348611828434864\n",
      "Cost after epoch 3150: 2.034808630378157\n",
      "Cost after epoch 3160: 2.0347549741627877\n",
      "Cost after epoch 3170: 2.0347001922903636\n",
      "Cost after epoch 3180: 2.0346442726722964\n",
      "Cost after epoch 3190: 2.0345872131080407\n",
      "Cost after epoch 3200: 2.0345290212936815\n",
      "Cost after epoch 3210: 2.034470885661257\n",
      "Cost after epoch 3220: 2.0344118274342518\n",
      "Cost after epoch 3230: 2.0343517462874052\n",
      "Cost after epoch 3240: 2.034290686211401\n",
      "Cost after epoch 3250: 2.0342287004527764\n",
      "Cost after epoch 3260: 2.0341658501640127\n",
      "Cost after epoch 3270: 2.034102203978124\n",
      "Cost after epoch 3280: 2.0340378376809585\n",
      "Cost after epoch 3290: 2.033972833864017\n",
      "Cost after epoch 3300: 2.0339072815435006\n",
      "Cost after epoch 3310: 2.033842557265626\n",
      "Cost after epoch 3320: 2.0337776291417833\n",
      "Cost after epoch 3330: 2.0337124479258604\n",
      "Cost after epoch 3340: 2.03364711802491\n",
      "Cost after epoch 3350: 2.033581748579161\n",
      "Cost after epoch 3360: 2.033516451918715\n",
      "Cost after epoch 3370: 2.0334513429769068\n",
      "Cost after epoch 3380: 2.0333865388336703\n",
      "Cost after epoch 3390: 2.033322158270886\n",
      "Cost after epoch 3400: 2.033258321324387\n",
      "Cost after epoch 3410: 2.0331963655151983\n",
      "Cost after epoch 3420: 2.033135294769696\n",
      "Cost after epoch 3430: 2.033075079285308\n",
      "Cost after epoch 3440: 2.033015833177\n",
      "Cost after epoch 3450: 2.0329576706414145\n",
      "Cost after epoch 3460: 2.0329007044832808\n",
      "Cost after epoch 3470: 2.0328450456200526\n",
      "Cost after epoch 3480: 2.0327908027386976\n",
      "Cost after epoch 3490: 2.032738081983726\n",
      "Cost after epoch 3500: 2.0326869866585926\n",
      "Cost after epoch 3510: 2.0326385717622966\n",
      "Cost after epoch 3520: 2.0325920117246676\n",
      "Cost after epoch 3530: 2.03254727739933\n",
      "Cost after epoch 3540: 2.032504453393902\n",
      "Cost after epoch 3550: 2.0324636215146263\n",
      "Cost after epoch 3560: 2.0324248594722927\n",
      "Cost after epoch 3570: 2.032388240572723\n",
      "Cost after epoch 3580: 2.0323538335678926\n",
      "Cost after epoch 3590: 2.032321702543274\n",
      "Cost after epoch 3600: 2.0322919068210274\n",
      "Cost after epoch 3610: 2.0322650502375224\n",
      "Cost after epoch 3620: 2.0322405986993255\n",
      "Cost after epoch 3630: 2.0322185211198187\n",
      "Cost after epoch 3640: 2.0321988552842596\n",
      "Cost after epoch 3650: 2.032181635351416\n",
      "Cost after epoch 3660: 2.032166890720138\n",
      "Cost after epoch 3670: 2.0321546458761803\n",
      "Cost after epoch 3680: 2.0321449203996456\n",
      "Cost after epoch 3690: 2.0321377290055\n",
      "Cost after epoch 3700: 2.032133081594842\n",
      "Cost after epoch 3710: 2.0321310712666363\n",
      "Cost after epoch 3720: 2.032131522273125\n",
      "Cost after epoch 3730: 2.0321344086136315\n",
      "Cost after epoch 3740: 2.0321397225012117\n",
      "Cost after epoch 3750: 2.032147453090491\n",
      "Cost after epoch 3760: 2.0321575854333855\n",
      "Cost after epoch 3770: 2.0321701004057764\n",
      "Cost after epoch 3780: 2.032184974792748\n",
      "Cost after epoch 3790: 2.0322021814025875\n",
      "Cost after epoch 3800: 2.0322216891860116\n",
      "Cost after epoch 3810: 2.032243119532467\n",
      "Cost after epoch 3820: 2.032266654940523\n",
      "Cost after epoch 3830: 2.03229228247548\n",
      "Cost after epoch 3840: 2.032319959794056\n",
      "Cost after epoch 3850: 2.0323496426330427\n",
      "Cost after epoch 3860: 2.0323812837996353\n",
      "Cost after epoch 3870: 2.032414833123102\n",
      "Cost after epoch 3880: 2.0324502375663496\n",
      "Cost after epoch 3890: 2.0324874413664578\n",
      "Cost after epoch 3900: 2.0325263861801304\n",
      "Cost after epoch 3910: 2.032566328755895\n",
      "Cost after epoch 3920: 2.032607751809309\n",
      "Cost after epoch 3930: 2.032650659341955\n",
      "Cost after epoch 3940: 2.0326949882448564\n",
      "Cost after epoch 3950: 2.032740674803079\n",
      "Cost after epoch 3960: 2.032787653724356\n",
      "Cost after epoch 3970: 2.0328358581194403\n",
      "Cost after epoch 3980: 2.032885219647044\n",
      "Cost after epoch 3990: 2.032935668691808\n",
      "Cost after epoch 4000: 2.032987134550418\n",
      "Cost after epoch 4010: 2.0330386532164857\n",
      "Cost after epoch 4020: 2.0330909132866672\n",
      "Cost after epoch 4030: 2.033143936492783\n",
      "Cost after epoch 4040: 2.0331976540079073\n",
      "Cost after epoch 4050: 2.03325199798384\n",
      "Cost after epoch 4060: 2.0333069006404574\n",
      "Cost after epoch 4070: 2.03336229428727\n",
      "Cost after epoch 4080: 2.033418111507052\n",
      "Cost after epoch 4090: 2.0334742853694516\n",
      "Cost after epoch 4100: 2.033530749648092\n",
      "Cost after epoch 4110: 2.033586472090643\n",
      "Cost after epoch 4120: 2.033642242578787\n",
      "Cost after epoch 4130: 2.0336981020855016\n",
      "Cost after epoch 4140: 2.033753992788888\n",
      "Cost after epoch 4150: 2.033809859518267\n",
      "Cost after epoch 4160: 2.0338656488506124\n",
      "Cost after epoch 4170: 2.0339213091045516\n",
      "Cost after epoch 4180: 2.0339767904834547\n",
      "Cost after epoch 4190: 2.0340320452372014\n",
      "Cost after epoch 4200: 2.0340870278163026\n",
      "Cost after epoch 4210: 2.034140765830259\n",
      "Cost after epoch 4220: 2.0341940582283473\n",
      "Cost after epoch 4230: 2.0342469662860703\n",
      "Cost after epoch 4240: 2.0342994564231835\n",
      "Cost after epoch 4250: 2.034351498633429\n",
      "Cost after epoch 4260: 2.0344030655093905\n",
      "Cost after epoch 4270: 2.0344541321301843\n",
      "Cost after epoch 4280: 2.03450467609027\n",
      "Cost after epoch 4290: 2.034554677543557\n",
      "Cost after epoch 4300: 2.0346041192383346\n",
      "Cost after epoch 4310: 2.0346521611390616\n",
      "Cost after epoch 4320: 2.034699546902113\n",
      "Cost after epoch 4330: 2.0347463541254878\n",
      "Cost after epoch 4340: 2.0347925762497585\n",
      "Cost after epoch 4350: 2.034838210086996\n",
      "Cost after epoch 4360: 2.0348832547954063\n",
      "Cost after epoch 4370: 2.0349277116878977\n",
      "Cost after epoch 4380: 2.034971584181975\n",
      "Cost after epoch 4390: 2.03501487777004\n",
      "Cost after epoch 4400: 2.035057599986952\n",
      "Cost after epoch 4410: 2.0350990531372575\n",
      "Cost after epoch 4420: 2.0351398975882002\n",
      "Cost after epoch 4430: 2.0351802195498623\n",
      "Cost after epoch 4440: 2.0352200332617105\n",
      "Cost after epoch 4450: 2.0352593555628884\n",
      "Cost after epoch 4460: 2.0352982048895756\n",
      "Cost after epoch 4470: 2.0353366010743374\n",
      "Cost after epoch 4480: 2.035374565284848\n",
      "Cost after epoch 4490: 2.035412119985985\n",
      "Cost after epoch 4500: 2.0354492889029543\n",
      "Cost after epoch 4510: 2.035485482214574\n",
      "Cost after epoch 4520: 2.0355212843276527\n",
      "Cost after epoch 4530: 2.0355567841446693\n",
      "Cost after epoch 4540: 2.0355920082774697\n",
      "Cost after epoch 4550: 2.035626985188251\n",
      "Cost after epoch 4560: 2.035661744246056\n",
      "Cost after epoch 4570: 2.0356963155484316\n",
      "Cost after epoch 4580: 2.0357307298761302\n",
      "Cost after epoch 4590: 2.035765018670127\n",
      "Cost after epoch 4600: 2.035799214009784\n",
      "Cost after epoch 4610: 2.03583277743221\n",
      "Cost after epoch 4620: 2.0358662500167\n",
      "Cost after epoch 4630: 2.0358997217361168\n",
      "Cost after epoch 4640: 2.0359332249126627\n",
      "Cost after epoch 4650: 2.0359667931662124\n",
      "Cost after epoch 4660: 2.036000460530792\n",
      "Cost after epoch 4670: 2.036034261298976\n",
      "Cost after epoch 4680: 2.0360682299918933\n",
      "Cost after epoch 4690: 2.0361024013495297\n",
      "Cost after epoch 4700: 2.036136810321427\n",
      "Cost after epoch 4710: 2.036170906395002\n",
      "Cost after epoch 4720: 2.0362052329079066\n",
      "Cost after epoch 4730: 2.0362398826351016\n",
      "Cost after epoch 4740: 2.036274889126836\n",
      "Cost after epoch 4750: 2.03631028684458\n",
      "Cost after epoch 4760: 2.0363461103298843\n",
      "Cost after epoch 4770: 2.036382394066555\n",
      "Cost after epoch 4780: 2.036419172461883\n",
      "Cost after epoch 4790: 2.0364564798465783\n",
      "Cost after epoch 4800: 2.036494350474715\n",
      "Cost after epoch 4810: 2.0365321602275777\n",
      "Cost after epoch 4820: 2.0365705036233623\n",
      "Cost after epoch 4830: 2.0366094797162355\n",
      "Cost after epoch 4840: 2.0366491204244177\n",
      "Cost after epoch 4850: 2.036689458317802\n",
      "Cost after epoch 4860: 2.0367305258326502\n",
      "Cost after epoch 4870: 2.036772355148272\n",
      "Cost after epoch 4880: 2.0368149781771754\n",
      "Cost after epoch 4890: 2.036858426572726\n",
      "Cost after epoch 4900: 2.0369027317365176\n",
      "Cost after epoch 4910: 2.036947141360737\n",
      "Cost after epoch 4920: 2.0369923473212044\n",
      "Cost after epoch 4930: 2.037038459098194\n",
      "Cost after epoch 4940: 2.0370855053334243\n",
      "Cost after epoch 4950: 2.037133515160744\n",
      "Cost after epoch 4960: 2.037182517459459\n",
      "Cost after epoch 4970: 2.0372325407411456\n",
      "Cost after epoch 4980: 2.037283613145047\n",
      "Cost after epoch 4990: 2.0373357624497728\n",
      "training time: 0:00:00.795473\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAf2klEQVR4nO3deZhddZ3n8ff3LrWltiRVqaQKkgoQlkpAwELgAZSGHtdWXHB0BlGb9mG0nW6w7RmXdnSUnqcfdGRcp3loUZuWVlSQppmxBQVHEIgkIUAWQgIJkJClslBbart1v/PHOZVciqrk1nLq3OXzep773HN/55x7v7+k6nzqLPd3zN0REZHylYi7ABERiZeCQESkzCkIRETKnIJARKTMKQhERMpcKu4Cpqqpqcnb29vjLkNEpKisXbt2v7s3TzSv6IKgvb2dNWvWxF2GiEhRMbMXJpunQ0MiImVOQSAiUuYUBCIiZU5BICJS5hQEIiJlTkEgIlLmFAQiImWubIJg+/5+vn7fFn67ZR/dAyNxlyMiUjCK7gtl0/X0rm6+++A2sg5mcFpLHe84cwkfOO9EFtVXxV2eiEhsrNhuTNPZ2enT/WZx/1CGJ196hbUvHOLhbftZvf0g1ekk//myU/j4m04mmbBZrlZEpDCY2Vp375xwXjkFwXjb9/dz4y+f4d827uHy0xfx3avOpSqdnJX3FhEpJMcKgrI5RzCR5U3zuPnq1/OVK1bywJZ9/OWPnyCbLa5gFBGZqbIOgjEfvrCdL7yjg/s27eUHj+yIuxwRkTmlIAhdc1E7l5++iK/+2zPsPHQ47nJEROaMgiBkZtzw7lW4w7d/sy3uckRE5oyCIEdrYzX/8fyl/HzdTnZ3D8RdjojInFAQjHPNRcsZzTp3PP5S3KWIiMwJBcE4SxfWcMmKJu54/CVdQSQiZUFBMIH3ntvG7u5B1u98Je5SREQipyCYwGWnt5BKGL/auCfuUkREIqcgmEBDdZoLT17I/Zv2xl2KiEjkFASTeNOpzTzf1c+e7sG4SxERiZSCYBIXnLQQgEef3x9zJSIi0VIQTKJjST0N1Wkefe5A3KWIiERKQTCJRMI4f/kC/rD9YNyliIhEKrIgMLMTzexBM9tkZhvN7LoJlrnUzLrNbH34+GJU9UzH2Usb2XHgMN2HdUczESldUd6hLAN82t3XmVkdsNbM7nf3TeOWe8jd/yTCOqbtrLZGAJ7a9QqXrGiOuRoRkWhEtkfg7rvdfV043QtsBtqi+rwonHlCAwBP7eyOuRIRkejMyTkCM2sHzgFWTzD7QjN70sx+aWYrJ1n/WjNbY2Zrurq6Iqz01Rqq0yxvmseTL+kbxiJSuiIPAjOrBe4Ernf3nnGz1wHL3P11wLeBuyd6D3e/xd073b2zuXluD9Gsamtg0+7xZYuIlI5Ig8DM0gQhcLu73zV+vrv3uHtfOP1/gbSZNUVZ01SduqiWnYcG6B/KxF2KiEgkorxqyIBbgc3uftMkyywOl8PM3hDWU1AX7p+6uA6Abfv6Yq5ERCQaUV41dBFwNfC0ma0P2z4PLAVw95uBK4FPmFkGGAA+6O4FNfbzqS1BEGzZ28vrTmyMuRoRkdkXWRC4+8OAHWeZ7wDfiaqG2bB0QQ2VqQRb9/bGXYqISCT0zeLjSCaMk5treXavDg2JSGlSEORhRUutzhGISMlSEOShfeE8dncPMJQZjbsUEZFZpyDIQ3tTDVmHlw4OxF2KiMisUxDkYdnCeQC8cKA/5kpERGafgiAP7WEQ7DhwOOZKRERmn4IgD/Nr0tRVpbRHICIlSUGQBzOjfeE87RGISElSEORp2cIa7RGISElSEORp2cIadh4aIDOajbsUEZFZpSDI0wnzaxjNOnt6BuMuRURkVikI8tTWWA3ArkP6LoGIlBYFQZ7a5odB8IqCQERKi4IgT9ojEJFSpSDIU1U6SVNthfYIRKTkKAimoK2xWkEgIiVHQTAFbfOrdWhIREqOgmAKxvYICuxumiIiM6IgmIK2xmqGMlkO9A/HXYqIyKxREExB2/waQFcOiUhpURBMwZFLSHXCWERKiIJgCo58qUx7BCJSQhQEU9BQnaauMqU9AhEpKQqCKWqbX81O7RGISAlREExRa2M1L2uPQERKiIJgivTtYhEpNQqCKWptrKZ7YIS+oUzcpYiIzAoFwRSNXTmkw0MiUioUBFOk4ahFpNREFgRmdqKZPWhmm8xso5ldd4xlzzOzjJldGVU9s2UsCHZqj0BESkQqwvfOAJ9293VmVgesNbP73X1T7kJmlgRuBO6LsJZZs6iuknTSdGhIREpGZHsE7r7b3deF073AZqBtgkX/ArgT2BdVLbMpkTCWNGg4ahEpHXNyjsDM2oFzgNXj2tuA9wB/f5z1rzWzNWa2pqurK6oy89baWKVLSEWkZEQeBGZWS/AX//Xu3jNu9jeAz7h79ljv4e63uHunu3c2NzdHVWre2hprdGhIREpGlOcIMLM0QQjc7u53TbBIJ/ATMwNoAt5uZhl3vzvKumaqrbGKvT2DjIxmSSd14ZWIFLfIgsCCrfutwGZ3v2miZdx9ec7yPwTuLfQQgOC7BFmHPd2DnLigJu5yRERmJMo9gouAq4GnzWx92PZ5YCmAu98c4WdHqq0xvEHNKwMKAhEpepEFgbs/DNgUlv9oVLXMttbGKkBfKhOR0qAD3NPQ2qhhJkSkdCgIpqEqnaSptlKXkIpISVAQTFObvksgIiVCQTBNbfN1XwIRKQ0KgmlqC+9U5u5xlyIiMiMKgmlqbaxmcCTLgf7huEsREZkRBcE06b4EIlIqFATTNPZFshcPHo65EhGRmVEQTFP7wnkA7NjfH3MlIiIzoyCYpuqKJIvrq9h+QEEgIsVNQTAD7U012iMQkaKnIJiB5U3z2HFA5whEpLgpCGagfeE8DvYP0z0wEncpIiLTpiCYgfYmnTAWkeKnIJiB5WNBoBPGIlLEFAQzsDT8LsF27RGISBFTEMxAVTpJW2O1gkBEipqCYIZOWVTL1r19cZchIjJtCoIZOm1xHdu6+siMZuMuRURkWhQEM3RqSx3DmSwvaMwhESlSCoIZOq2lDoBn9/TGXImIyPQoCGbolEW1mMGWvQoCESlOCoIZqq5IsmxBDc8qCESkSCkIZsGpLXVs0aEhESlSCoJZcNriOnYcOMzgyGjcpYiITJmCYBactriO0ayzbZ++TyAixUdBMAtWtTYAsGFXd8yViIhMXV5BYGbvz6etXC1dUENdZYoNLysIRKT45LtH8Lk828pSImF0tNazYVdP3KWIiExZ6lgzzextwNuBNjP7Vs6seiBznHVPBG4DWgAHbnH3b45b5grgBiAbvt/17v7wVDtRCFa1NfCjx14gM5olldQRNxEpHscMAuBlYA3wLmBtTnsv8KnjrJsBPu3u68ysDlhrZve7+6acZX4D3OPubmZnAT8FTp9SDwrEmW0NDGWyPNfVz2mL6+IuR0Qkb8cMAnd/EnjSzP7Z3UcAzGw+cKK7HzrOuruB3eF0r5ltBtqATTnL5F5mM49gz6EorWqrB+DpXd0KAhEpKvkew7jfzOrNbAGwDvgHM/tf+X6ImbUD5wCrJ5j3HjN7Bvg/wDWTrH+tma0xszVdXV35fuycWt5US3U6qSuHRKTo5BsEDe7eA7wXuM3dzwcuz2dFM6sF7iQ4/v+as6nu/gt3Px14N8H5gtdw91vcvdPdO5ubm/MseW4lwxPGG3XlkIgUmXyDIGVmS4B/D9yb75ubWZogBG5397uOtay7/w44ycya8n3/QnNmWwMbX+4hmy3aI1wiUobyDYKvAL8CnnP3x83sJGDrsVYwMwNuBTa7+02TLHNKuBxmdi5QCRzIt/hCs7K1nsPDozyvW1eKSBE53lVDALj7z4Cf5bx+HnjfcVa7CLgaeNrM1odtnweWhu9xc/geHzazEWAA+IC7F+2f06vagm8Yb3y5m1MW1cZcjYhIfvIKAjM7Afg2wcYd4CHgOnffOdk64fcB7Fjv6+43AjfmV2rhO2VRLRWpBBtf7uGKs9viLkdEJC/5Hhr6AXAP0Bo+/jVskxzpZIIzFtfpyiERKSr5BkGzu//A3TPh44dAYV6+E7OO1gY27OqmiI9wiUiZyTcIDpjZh8wsGT4+RBGf1I3SqrZ6egYz7Dw0EHcpIiJ5yTcIriG4dHQPwbeFrwQ+GlFNRU1DUotIsZnK5aMfcfdmd19EEAxfjq6s4nXa4jqSCWPjyxqJVESKQ75BcFbu2ELufpBgyAgZpyqdZMWiWt2bQESKRr5BkAgHmwMgHHMor0tPy9GqNp0wFpHikW8QfB141MxuMLMbgEeAr0ZXVnFb1VrP/r5h9vUOxV2KiMhx5fvN4tvMbA1wWdj03nH3FZAcK9uOnjBuqa+KuRoRkWPL+/BOuOHXxj8PZyypxww27Orh8jNa4i5HROSYdE/FCNRWpljeNE8njEWkKCgIIrKqtYGN+i6BiBQBBUFEVrbW83L3IIf6h+MuRUTkmBQEEeloDe5hvHm3vlgmIoVNQRCRM5YEQbBJQSAiBU5BEJGm2kpa6ivZpKEmRKTAKQgi1LGkXnsEIlLwFAQRWtnawLZ9fQyOjMZdiojIpBQEEeporSeTdbbt64u7FBGRSSkIItQxdsJY5wlEpIApCCK0dEEN8yqSOk8gIgVNQRChRMI4Y0k9GzXUhIgUMAVBxDpa69m8u5dsVvcmEJHCpCCIWMeSevqGMrx06HDcpYiITEhBELGxoSZ0wlhECpWCIGKntgQ3s9cJYxEpVAqCiFWlk5zSXKs9AhEpWAqCOdDRqqEmRKRwKQjmQMeSenZ3D3JQ9yYQkQIUWRCY2Ylm9qCZbTKzjWZ23QTLXGVmT5nZ02b2iJm9Lqp64qR7E4hIIYtyjyADfNrdO4ALgE+aWce4ZbYDb3L3M4EbgFsirCc2Z2ioCREpYKmo3tjddwO7w+leM9sMtAGbcpZ5JGeVx4AToqonTgvmVbCkoUrnCUSkIM3JOQIzawfOAVYfY7E/A345yfrXmtkaM1vT1dU1+wXOgQ4NNSEiBSryIDCzWuBO4Hp3n/BPYjP7I4Ig+MxE8939FnfvdPfO5ubm6IqNUEdrPc919eveBCJScCINAjNLE4TA7e5+1yTLnAV8D7jC3Q9EWU+cOpbUM5p1nt3bG3cpIiKvEuVVQwbcCmx295smWWYpcBdwtbs/G1UthWBlawOgE8YiUngiO1kMXARcDTxtZuvDts8DSwHc/Wbgi8BC4H8HuUHG3TsjrCk2J8yvpq4ypRPGIlJworxq6GHAjrPMx4CPRVVDIRm7N4H2CESk0OibxXMouDdBj+5NICIFRUEwhzqW1NM/PMqLB3VvAhEpHAqCOXTk3gQ6TyAiBURBMIdOWVRLKmE6TyAiBUVBMIeq0klOWVSrPQIRKSgKgjnWoSuHRKTAKAjmWEdrPXt6BjnQNxR3KSIigIJgzumEsYgUGgXBHOvQvQlEpMAoCOZYY00FbY3V2iMQkYKhIIiBhpoQkUKiIIjBytZ6nuvqo28oE3cpIiIKgji8ftl8sg7rX3wl7lJERBQEcThnaSMJgzUvHIy7FBERBUEc6qrSnL64njU7DsVdioiIgiAune3zeeLFQ2RGs3GXIiJlTkEQk872BfQPj/LMHt3DWETipSCISeey+QA8vkPnCUQkXgqCmLQ2VtPWWM2aF3SeQETipSCI0Xnt81n9/AHdulJEYqUgiNHFK5rZ3zes8wQiEisFQYwuWdEEwENbu2KuRETKmYIgRi31VZzWUsdDW/fHXYqIlDEFQcwuWdHEH3YcZGB4NO5SRKRMKQhidvGKJoYzWf6gy0hFJCYKgpidv3whFakEv92yL+5SRKRMKQhiVl2R5OJTmrh/017cdRmpiMw9BUEBeMvKFnYeGtBdy0QkFgqCAnD5GS0kDO7buDfuUkSkDEUWBGZ2opk9aGabzGyjmV03wTKnm9mjZjZkZn8dVS2Frqm2ks5lC/jVxj1xlyIiZSjKPYIM8Gl37wAuAD5pZh3jljkI/CXwPyOsoyi8eWULz+zpZcf+/rhLEZEyE1kQuPtud18XTvcCm4G2ccvsc/fHgZGo6igW7zhrCWZw9/pdcZciImVmTs4RmFk7cA6weprrX2tma8xsTVdXaQ7HsKShmgtPWsgvntilq4dEZE5FHgRmVgvcCVzv7tO6LMbdb3H3TnfvbG5unt0CC8h7zmnjhQOHWaeb2ovIHIo0CMwsTRACt7v7XVF+Vil425lLqEonuGvdzrhLEZEyEuVVQwbcCmx295ui+pxSUluZ4q0rF3PP+pfpH8rEXY6IlIko9wguAq4GLjOz9eHj7Wb2cTP7OICZLTazncBfAV8ws51mVh9hTQXv6gvb6R3KcNcTOmksInMjFdUbu/vDgB1nmT3ACVHVUIzOXdrImW0N3PbIDj50/lKCHSsRkejom8UFxsz48IXL2Lqvj99vOxB3OSJSBhQEBeidr2tlUV0l33lwa9yliEgZUBAUoKp0kk9cejKPPX+QR5/TXoGIREtBUKD+wxuWsqiukm/8+ll9wUxEIqUgKFBV6SR/funJrN5+kAee0U1rRCQ6CoICdtUFyzhlUS1fuXcTQxnd01hEoqEgKGDpZIIvvbODFw4c5nsPbY+7HBEpUQqCAnfJimbetmox3/z1Vrbs6Y27HBEpQQqCInDDu1dRV5XiU3esZziTjbscESkxCoIi0FRbyd+990w27e7hK/dujLscESkxCoIi8eaVi7n2jSfxo8de5J8eeyHuckSkhEQ21pDMvs+89XS27u3lS/+ygfqqFFec3Xb8lUREjkN7BEUkmTC+e9W5nNe+gE/dsZ5fPKH7FojIzCkIikxNRYrvf/Q83rB8AZ+640m+ft8Wsll981hEpk+HhorQvMoUt11zPv/t7g18+4FtrN5+kK9deRbLFs6Lu7RZ4+4Mj2YZGB7lcPgIpjMcHgmm+4cyDIzkzs/kLDdK/3DmyHSwXPA6mTDSyQTpZIKKVIJ00qhOJ6mvTlNflQ6eq1M0VKdpqq1kUV0li+qqWFRfyYKaChIJDQ0upcWKbRybzs5OX7NmTdxlFAR35+drd/KVezcxnMly9QXL+E9vOpnmuso5rWM4k6V/KEPfuMfhoXDjO8HG+vDYRj1nwz22sR6bPzrFPZ3qdJKaiiTVFWPPKWrSSeZVHp2urkiSdWdkNMtwZuw5y+GRUXoHR+geGKFnIHgeGX3t56cSFoRDfRAOLfWVLK6voqU+CIqWcHp+TVr3kpCCYmZr3b1zwnkKguK3u3uAr9/3LHet20kyYfzxGS2863WtXHjyQhprKqb8fv1DGfb3DbG/b4iu3iG6+obp6j36en/fEAf7h+kbzNA7lJnSdxvy3VjX5Myfd2T51KvWrUmnqK4I1q1KJWf1L3V3Z2BklP29w+zrHWRf7xD7egbp6htiX88Qe8PXe3sGOXR45DXrVyQTOcEwFhpVLG6opKWuikVhe21lSoEhc0JBUCae7+rjR4+9yN3rd3GwfxgzOLm5luVN82hrrKauKkVNRYqsO8OZLEOZLIf6hznQP8SB/mEO9A2zv2+Iw8OvHdfIDBbUVNBcV0lTbSUL5lVQV5WitipFbUX4XBk+ws+ZVxntxrpQDGVG2dczxL7eQfb2DLGne5C9vYNBYIRhsa9niN4J7kNdlU7QUJ2mripNfVUq5/BUivqqNPMqU1Slk1SlE1SlklSlk1RXBNOV6STVY/PSySPLVaaSJEvw3zku7k4me/R3Zmwvcng0mB7J+NHpcN7IaJbhUWckc7Q9WNePLndk3VFGxvZOj7yPv/ozwmXf33kCH7vkpGn1Q0FQZkZGszz50iv8ftsBNu3uZvv+fna/MkjfcIbc/+500phfU8HC2kqaaitYMK+CptrKIxv74LmC5nDDn0rq2oKZ6BvKhHsRY6ExSFfvED0DGXoGR+gdDJ57BkboGczQMzBCZpoXAqSTdiQsjgZFEBK5oVI5Ni81bn7OepWpBJXhc9UxnlMJi2TvJpt1hjJZhjKjwfPI0enBkdGj80ayE7YNhxvhoZHRYHoky9Do0fcZ28APZbIMh+97tC2YP9vXY5gFe40VyQTp8DxVcL4qbEsGbWPnscba3rKqhfecM727+x4rCHSyuASlkwk62xfQ2b7gVe1jhzsSZlQkEyX513khq61MUdtcy0nNtXkt7+5HNnyDmVEGR4LzKIMj2SPTQzmvB0dGGcyMTQcbscGRYAM4mMlZb3iUQ/0jR+fnzJtu8AAkjCPBkEwkSBgkzEhYcAvWRGLstTGWF6NZJzPqwXPWGc1mGc3mvvYZ1TRWV2UqCL2KZOLoc05bfXX6yLzK3GXC/uSuV5FKvmrDnXvRwdGNeM4GPJVzcULYXmh7bAqCMmJm1FTov7xYmNmRQz4NpOfkMzOjWQYzY+GRPRI040Pj6F/gr/7LfOx5NOtkPfhrPuvBtPvR6aw7DqQTRjKRIJUwkkkjaUYyYa96XZEKN9qp3D2Uo3svY/Mq02OHzHLaUgntyeZBWwUROSKVTFCbTFBbqU1DOVFUioiUOQWBiEiZUxCIiJQ5BYGISJlTEIiIlDkFgYhImVMQiIiUOQWBiEiZK7qxhsysC5juTXubgP2zWE4xUJ/Lg/pcHmbS52Xu3jzRjKILgpkwszWTDbpUqtTn8qA+l4eo+qxDQyIiZU5BICJS5sotCG6Ju4AYqM/lQX0uD5H0uazOEYiIyGuV2x6BiIiMoyAQESlzZRMEZvZWM9tiZtvM7LNx1zMTZvZ9M9tnZhty2haY2f1mtjV8nh+2m5l9K+z3U2Z2bs46HwmX32pmH4mjL/kwsxPN7EEz22RmG83surC9lPtcZWZ/MLMnwz5/OWxfbmarw77dYWYVYXtl+HpbOL89570+F7ZvMbO3xNOj/JlZ0syeMLN7w9cl3Wcz22FmT5vZejNbE7bN7c+2u5f8A0gCzwEnARXAk0BH3HXNoD9vBM4FNuS0fRX4bDj9WeDGcPrtwC8BAy4AVoftC4Dnw+f54fT8uPs2SX+XAOeG03XAs0BHiffZgNpwOg2sDvvyU+CDYfvNwCfC6T8Hbg6nPwjcEU53hD/vlcDy8PcgGXf/jtP3vwL+Gbg3fF3SfQZ2AE3j2ub0Z7tc9gjeAGxz9+fdfRj4CXBFzDVNm7v/Djg4rvkK4B/D6X8E3p3TfpsHHgMazWwJ8Bbgfnc/6O6HgPuBt0Zf/dS5+253XxdO9wKbgTZKu8/u7n3hy3T4cOAy4Odh+/g+j/1b/By43MwsbP+Juw+5+3ZgG8HvQ0EysxOAdwDfC18bJd7nSczpz3a5BEEb8FLO651hWylpcffd4fQeoCWcnqzvRflvEu7+n0PwF3JJ9zk8RLIe2Efwi/0c8Iq7Z8JFcus/0rdwfjewkCLrM/AN4L8C2fD1Qkq/zw7cZ2ZrzezasG1Of7Z1h+oS5O5uZiV3XbCZ1QJ3Ate7e0/wx1+gFPvs7qPA2WbWCPwCOD3mkiJlZn8C7HP3tWZ2adz1zKGL3X2XmS0C7jezZ3JnzsXPdrnsEewCTsx5fULYVkr2hruIhM/7wvbJ+l5U/yZmliYIgdvd/a6wuaT7PMbdXwEeBC4kOBQw9gdcbv1H+hbObwAOUFx9vgh4l5ntIDh8exnwTUq7z7j7rvB5H0Hgv4E5/tkulyB4HFgRXn1QQXBi6Z6Ya5pt9wBjVwp8BPiXnPYPh1cbXAB0h7ucvwLebGbzwysS3hy2FZzwuO+twGZ3vylnVin3uTncE8DMqoF/R3Bu5EHgynCx8X0e+7e4EnjAg7OI9wAfDK+wWQ6sAP4wN72YGnf/nLuf4O7tBL+jD7j7VZRwn81snpnVjU0T/ExuYK5/tuM+Yz5XD4Kz7c8SHGf9m7jrmWFffgzsBkYIjgX+GcGx0d8AW4FfAwvCZQ34btjvp4HOnPe5huBE2jbgT+Pu1zH6ezHBcdSngPXh4+0l3uezgCfCPm8Avhi2n0SwUdsG/AyoDNurwtfbwvkn5bzX34T/FluAt8Xdtzz7fylHrxoq2T6HfXsyfGwc2zbN9c+2hpgQESlz5XJoSEREJqEgEBEpcwoCEZEypyAQESlzCgIRkTKnIJCSZ2Z/Z2Z/ZGbvNrPPTXHd5nBkyyfM7JKoapzks/uOv5TIzCkIpBycDzwGvAn43RTXvRx42t3PcfeHZr0ykQKgIJCSZWZfM7OngPOAR4GPAX9vZl+cYNl2M3sgHOP9N2a21MzOJhgO+IpwrPjqceu83sz+XzhY2K9yhgT4rZl9M1xng5m9IWxfYGZ3h5/xmJmdFbbXmtkPLBiT/ikze1/OZ/wPC+5J8JiZtYRt7w/f90kzm2qwibxW3N+s00OPKB8EIfBtgmGcf3+M5f4V+Eg4fQ1wdzj9UeA7EyyfBh4BmsPXHwC+H07/FviHcPqNhPeNCOv4Ujh9GbA+nL4R+EbOe88Pnx14Zzj9VeAL4fTTQFs43Rj3v7Eexf/Q6KNS6s4l+Pr+6QRj9UzmQuC94fQ/EWx4j+U0YBXBaJEQ3Pxod878H0Nw7wgzqw/HDboYeF/Y/oCZLTSzeuCPCcbWIZx3KJwcBu4Np9cSjDcE8Hvgh2b2U2BsAD6RaVMQSEkKD+v8kGAUxv1ATdBs64EL3X1gph8BbHT3CyeZP37slumM5TLi7mPrjRL+vrr7x83sfIIbuKw1s9e7+4FpvL8IoHMEUqLcfb27n83R21o+ALzF3c+eJAQe4ehf5VcBxzsxvAVoNrMLIRgm28xW5sz/QNh+McEIkd3he14Vtl8K7Hf3HoKbznxybMVw9MhJmdnJ7r7a3b8IdPHq4YdFpkx7BFKyzKwZOOTuWTM73d03HWPxvwB+YGb/hWDj+qfHem93HzazK4FvmVkDwe/SNwhGkAQYNLMnCM4lXBO2/Xfg++EJ7MMcHWb4b4HvmtkGgr/8v8yxD/l8zcxWEOyV/Ibg0JfItGn0UZFZZma/Bf7a3dfEXYtIPnRoSESkzGmPQESkzGmPQESkzCkIRETKnIJARKTMKQhERMqcgkBEpMz9fxo49LJdZpeyAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "paras = skipgram_model_training(X, Y_one_hot, vocab_size, 50, 0.05, 5000, batch_size=128, parameters=None, print_cost=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.150655Z",
     "start_time": "2019-07-14T06:37:42.131158Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['WRD_EMB', 'W'])\n"
     ]
    }
   ],
   "source": [
    "print(paras.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.184048Z",
     "start_time": "2019-07-14T06:37:42.152909Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13, 50)\n",
      "[[-1.94240532e+00  1.25388014e+00  8.52578571e-01 -2.98138017e+00\n",
      "  -2.58780717e+00 -9.71020430e-04 -1.73553609e+00  1.12999026e-01\n",
      "  -2.35643225e+00  1.95908586e+00  2.76110961e+00 -5.44402389e-01\n",
      "   1.71649252e-02 -1.34423255e+00 -1.94388373e+00 -3.41736748e+00\n",
      "   1.36062969e+00  1.84118486e+00 -9.37857311e-01 -3.14939284e+00\n",
      "   3.70016337e+00  2.75222266e-01  4.01376679e+00  2.31607604e+00\n",
      "  -3.78692573e+00 -6.96532942e-01  1.23531761e+00 -1.96679841e+00\n",
      "  -1.59796159e+00  3.76965112e-01 -4.35389891e-01 -2.64731268e+00\n",
      "  -2.48749107e+00  2.62133150e+00 -2.77336286e+00 -5.56669983e-01\n",
      "   3.81531447e-01 -3.20986215e+00  7.25909928e-02 -1.77044623e+00\n",
      "  -1.99152807e+00  1.61951290e+00  1.42550694e+00  7.80641078e-01\n",
      "   2.23890659e+00  1.62618909e+00 -1.07357472e+00  3.71057956e-01\n",
      "  -2.29688274e+00 -3.24455311e+00]\n",
      " [ 9.31320328e-01 -2.91464645e+00 -2.47248438e-01  1.70027301e+00\n",
      "   1.08019738e-01  3.72189306e-02 -4.43406387e-01  4.65893139e-01\n",
      "   1.18804537e+00  1.68441324e+00  2.41793984e-01 -1.80730373e+00\n",
      "   1.95618056e+00  3.51174412e-01  6.49899658e-01 -6.08598800e-01\n",
      "  -3.57159793e+00 -5.62879062e+00 -5.79523762e-01  4.65478946e+00\n",
      "  -5.03911709e+00  1.53430467e+00 -3.67931177e+00  5.28867891e-01\n",
      "   1.56664711e+00 -1.83896759e+00  1.18149977e+00  1.22847993e+00\n",
      "  -1.61550196e+00 -1.43678564e+00 -4.97218455e-02  1.24152151e+00\n",
      "   3.45704074e+00  1.76861464e-02  3.20344750e-01 -2.09190420e+00\n",
      "   1.21320038e+00  1.35228631e+00 -8.37334535e-01  1.71375334e+00\n",
      "   2.51982720e+00 -7.44650621e-01  6.19279191e-01 -2.60730108e+00\n",
      "  -4.38619882e+00 -1.54267522e-01 -8.64768005e-01  1.13926211e+00\n",
      "   4.85163140e+00  2.42701228e+00]\n",
      " [ 1.39463536e+00 -2.97841917e+00 -4.56553126e-02 -3.96717126e+00\n",
      "  -1.92248906e+00 -6.36879623e-01 -2.89349016e+00 -8.94032795e-01\n",
      "  -1.17950175e+00  1.77208851e+00 -5.45201883e-01  1.27891476e+00\n",
      "   5.87705629e-01  2.62415338e-01 -2.60567035e+00  1.00950170e+00\n",
      "   3.31207076e+00  3.90934648e+00  2.45856113e+00  1.91122779e+00\n",
      "   2.88642489e+00 -1.04671305e+00 -1.47419025e-01 -1.15475099e+00\n",
      "  -8.13082290e-01 -7.56431861e-01 -1.87534052e+00 -3.26026437e-01\n",
      "  -3.97427111e-01 -1.53081467e+00  7.66874336e-01 -2.00092270e+00\n",
      "  -2.05884365e+00 -4.38261432e-01 -5.27150427e-01 -2.05045374e+00\n",
      "  -2.24242979e-01 -1.39927462e+00  2.21694789e-01  1.66905420e+00\n",
      "   1.20011162e-01 -1.36087152e+00 -1.74866576e+00 -8.23645870e-01\n",
      "   4.34996331e-01 -3.98266779e+00 -1.27670899e+00 -1.27987783e+00\n",
      "   8.41419988e-01 -1.63535856e+00]\n",
      " [-1.92110493e-01 -2.97881579e+00 -2.08946648e+00 -3.86039186e-01\n",
      "  -8.17910465e-01 -2.89871081e-01 -8.24247587e-01  2.18399942e+00\n",
      "  -4.88458381e-01  2.70012576e+00  2.66142197e-01 -1.47951430e+00\n",
      "   1.89355791e+00 -7.15493374e-01  1.09400639e+00 -9.61297240e-01\n",
      "  -1.31750078e+00 -2.48118026e+00 -6.16612445e-01  3.92741647e+00\n",
      "  -2.31728763e+00  7.43102505e-01 -2.57777602e+00  1.05143139e+00\n",
      "   1.60317945e+00 -1.69298297e+00  9.30660711e-01  1.42084754e+00\n",
      "  -1.42957059e+00 -1.24136663e-01 -5.26140838e-01  3.41113933e-01\n",
      "   1.80746335e+00  4.55544607e-01  5.71196164e-01 -2.01806390e+00\n",
      "   5.73621781e-01  1.99172648e-01 -1.32821111e+00  1.17648349e+00\n",
      "   1.38612949e+00 -1.15401350e+00  3.71867378e-01 -3.04302864e+00\n",
      "  -3.22314895e+00 -2.61658425e+00 -3.73690908e-01 -1.14364124e-01\n",
      "   4.75398984e+00  1.80892248e+00]\n",
      " [ 2.01715907e+00 -3.41397315e+00  2.14756102e-01 -2.66333316e+00\n",
      "  -1.26119355e+00 -5.24124195e-01 -2.51353316e+00 -1.34324198e+00\n",
      "   8.39736013e-03  1.87994596e+00 -6.42786002e-01  7.62532225e-01\n",
      "   9.81787885e-01  4.22594156e-01 -2.79738568e+00  1.39784496e+00\n",
      "   2.26420348e+00  1.70830281e+00  2.37544184e+00  2.83716279e+00\n",
      "   1.17016937e+00 -6.93489562e-01 -1.44207474e+00 -1.34446058e+00\n",
      "  -1.47998599e-01 -1.32481124e+00 -1.59120825e+00  4.85622730e-01\n",
      "  -5.79737099e-01 -1.88949585e+00  5.41926487e-01 -1.13301573e+00\n",
      "  -9.90242813e-01 -3.86294556e-01  3.43989462e-02 -2.38548779e+00\n",
      "   2.16143308e-01 -9.02538090e-01  3.34817357e-01  2.75701949e+00\n",
      "   1.82075681e+00 -1.37643523e+00 -1.64317575e+00 -1.20312976e+00\n",
      "  -5.00116734e-01 -3.73083918e+00 -1.48213763e+00 -1.04523317e+00\n",
      "   1.90077196e+00 -1.14596814e+00]\n",
      " [-3.07361902e+00  3.93452239e+00 -3.80592027e-01  1.06855147e+00\n",
      "   2.88915411e+00  7.99067601e-01  3.89043431e+00  2.25476875e+00\n",
      "   1.16475454e-01 -1.78070814e+00  8.32231227e-01 -1.19872640e+00\n",
      "  -1.07982832e+00 -1.84624694e+00  2.07079430e+00 -2.36642855e+00\n",
      "  -9.46916402e-01 -1.16166668e+00 -2.92006125e+00 -5.37659346e+00\n",
      "   2.22792875e-01  1.43288010e+00  3.03775726e+00  1.17969930e+00\n",
      "  -1.43134096e-01  1.20321755e+00  3.06191763e+00 -5.65876873e-01\n",
      "   1.75050043e+00  4.18478383e+00 -2.51438864e+00 -5.92010224e-02\n",
      "  -7.56112372e-01  2.14053594e+00  7.57722445e-01  2.78678476e+00\n",
      "  -4.55454479e-01 -5.96661807e-01 -3.36772281e-01 -2.18361177e+00\n",
      "  -1.67685445e+00  1.46542204e+00  1.80302029e+00  2.74417270e+00\n",
      "   2.52461169e+00  2.16860638e+00  1.11694750e+00  1.07898538e+00\n",
      "  -3.01294130e+00  6.90807700e-01]\n",
      " [-3.39523825e+00  3.87696546e+00 -3.12129085e+00  3.08947656e+00\n",
      "  -4.46984283e-01  2.83029823e+00  1.03918453e+00  6.67979429e-01\n",
      "  -2.24724966e-01  2.16505353e+00  2.87940488e-01 -3.35240749e+00\n",
      "  -3.03723736e+00 -3.87306741e+00  8.27308676e-01 -2.46364061e+00\n",
      "  -7.68187842e-01  2.62947253e-01 -4.19225715e+00 -5.13802465e+00\n",
      "   1.83371384e-01 -1.23651062e+00  1.03408896e-01  3.84376200e+00\n",
      "  -2.63539143e+00 -9.96220904e-01  1.27793748e+00  4.59405353e+00\n",
      "   1.49329086e+00  4.56292601e+00 -2.99689749e+00  1.69205049e+00\n",
      "   5.73745189e-01  2.29388116e+00 -1.77392174e+00  3.35831690e+00\n",
      "   3.48640601e-01 -1.37834600e+00  6.36119719e-01  2.63806963e-01\n",
      "   5.93603784e-01 -1.18097177e+00  2.15221285e+00 -2.10264742e+00\n",
      "  -9.59559723e-01  1.42253042e+00  2.69159796e+00 -2.76248434e-01\n",
      "  -2.48767359e+00 -2.09015175e+00]\n",
      " [ 7.69215448e-01 -2.85620366e+00 -2.13942711e-01 -2.52123782e+00\n",
      "  -3.04523904e+00  2.50247198e-01 -3.61616055e+00 -1.26782959e+00\n",
      "  -9.09550677e-01  3.62685095e+00  5.94773551e-01 -6.47302087e-01\n",
      "   5.10292746e-01 -5.88284101e-01 -3.12872392e+00 -7.65648337e-01\n",
      "   1.43468466e+00  1.38622962e+00  8.35041749e-01  2.08507094e+00\n",
      "   1.19348996e+00 -6.17586371e-01 -1.03013392e+00  8.14620444e-01\n",
      "  -2.23393309e+00 -2.29906305e+00 -8.72293324e-01  8.64451651e-01\n",
      "  -1.45063034e+00 -1.47491652e+00  8.70389990e-02 -1.48171149e+00\n",
      "  -5.10014137e-01  7.93947138e-01 -2.01656519e+00 -2.34552135e+00\n",
      "   8.69772450e-01 -1.84491034e+00  3.67107339e-01  2.33355152e+00\n",
      "   1.24042904e+00 -1.47146190e+00 -4.43043273e-01 -2.48730177e+00\n",
      "  -1.43294071e+00 -2.32519921e+00 -1.42897013e+00 -6.10911051e-01\n",
      "   1.58529404e+00 -2.21942401e+00]\n",
      " [ 1.62528129e-01 -3.67429425e+00 -1.27524802e+00 -3.88570808e+00\n",
      "  -5.86628474e-01 -8.40334855e-01 -1.30464955e+00  1.12437346e+00\n",
      "  -7.91840570e-01  1.69886279e+00  1.07190671e-01  6.70204897e-01\n",
      "   1.86557583e+00 -2.44131328e-01  6.93922808e-01 -6.49772844e-01\n",
      "   1.86396260e-01  1.37890823e+00  9.12153375e-01  4.05379299e+00\n",
      "  -6.15383388e-01  2.43610369e-02 -4.79295937e-01 -1.52248202e+00\n",
      "   1.95464234e-01  6.83104909e-01 -4.84302467e-01 -2.51043730e+00\n",
      "  -1.02013973e+00 -1.17693764e+00 -9.26853880e-02 -2.49732688e+00\n",
      "  -4.70623535e-01 -1.39379537e+00 -1.11923235e+00 -1.64019407e+00\n",
      "  -9.47263786e-01  6.80042470e-01 -4.93403417e-01  2.15915808e-01\n",
      "  -1.29725286e+00 -1.27868726e+00 -1.95768050e+00 -7.40460472e-01\n",
      "  -1.55051229e+00 -3.06288043e+00 -6.57528553e-01  9.78349293e-01\n",
      "   2.34677861e+00  1.42658047e+00]\n",
      " [-1.88256617e-01 -2.97839909e+00 -2.06912118e+00 -3.87545045e-01\n",
      "  -8.23020732e-01 -2.84985214e-01 -8.37182610e-01  2.17692909e+00\n",
      "  -4.90592531e-01  2.69672902e+00  2.75653653e-01 -1.48649470e+00\n",
      "   1.89426461e+00 -7.09622212e-01  1.08854897e+00 -9.78363446e-01\n",
      "  -1.32897122e+00 -2.49576744e+00 -6.21193332e-01  3.92380856e+00\n",
      "  -2.32501697e+00  7.55233054e-01 -2.56696460e+00  1.05880660e+00\n",
      "   1.59214149e+00 -1.70131730e+00  9.40510049e-01  1.40366280e+00\n",
      "  -1.43545780e+00 -1.34757782e-01 -5.21036566e-01  3.31540602e-01\n",
      "   1.81879189e+00  4.64158229e-01  5.52693195e-01 -2.02102401e+00\n",
      "   5.82199160e-01  1.98805909e-01 -1.32885956e+00  1.17011705e+00\n",
      "   1.36958511e+00 -1.15073272e+00  3.81603162e-01 -3.03535004e+00\n",
      "  -3.22891013e+00 -2.58966378e+00 -3.81901396e-01 -1.01855936e-01\n",
      "   4.74907349e+00  1.81153398e+00]\n",
      " [-2.30457857e+00  3.34019812e+00 -3.29186818e+00  3.34746587e+00\n",
      "   2.33696987e-01  1.38820066e+00  1.99417680e+00  1.25427582e+00\n",
      "  -3.22939471e-01 -7.71960541e-01 -9.50566029e-01 -1.25214438e+00\n",
      "  -2.20062607e+00 -2.06339082e+00  5.05852267e+00 -5.22918226e-01\n",
      "  -2.54698269e+00  9.90676011e-01 -3.39793880e+00 -1.37830009e+00\n",
      "  -2.07439058e+00 -1.88700723e+00 -5.98044049e-01  1.28629066e+00\n",
      "  -1.03861837e-01  2.86889148e+00 -5.39855059e-01  1.60350442e+00\n",
      "   1.05929324e+00  2.52250470e+00 -6.08896557e-01  1.95401171e+00\n",
      "   1.76726664e+00 -1.97430671e+00 -1.71479068e+00  4.19578314e+00\n",
      "  -1.73269487e+00  2.46530130e+00  2.65186881e-01 -2.66352273e+00\n",
      "  -2.18956813e+00 -6.92029804e-01  2.20648180e-01 -1.16671096e+00\n",
      "  -2.26806105e+00  2.58518663e+00  4.13726510e+00  8.72637756e-01\n",
      "  -1.89607119e+00  1.07116074e+00]\n",
      " [-1.86823990e-01 -2.98150105e+00 -2.08449714e+00 -3.88513324e-01\n",
      "  -8.27035625e-01 -2.90601262e-01 -8.24498131e-01  2.18558745e+00\n",
      "  -4.84189200e-01  2.70201410e+00  2.70211259e-01 -1.48815468e+00\n",
      "   1.89298531e+00 -7.18898027e-01  1.10068189e+00 -9.72968518e-01\n",
      "  -1.32602078e+00 -2.48598975e+00 -6.31374048e-01  3.94260423e+00\n",
      "  -2.32924464e+00  7.47557713e-01 -2.58208850e+00  1.05478402e+00\n",
      "   1.60584935e+00 -1.69335892e+00  9.40509161e-01  1.41234012e+00\n",
      "  -1.43210930e+00 -1.22184981e-01 -5.26742986e-01  3.32025099e-01\n",
      "   1.82216795e+00  4.55946045e-01  5.59786490e-01 -2.01780739e+00\n",
      "   5.68832505e-01  2.03730299e-01 -1.32517755e+00  1.17758717e+00\n",
      "   1.37454043e+00 -1.15903296e+00  3.73721500e-01 -3.04056124e+00\n",
      "  -3.23590569e+00 -2.61246802e+00 -3.71765964e-01 -1.06752779e-01\n",
      "   4.75917018e+00  1.81812608e+00]\n",
      " [ 1.38209838e+00 -2.99272235e+00 -6.37205185e-02 -3.96715158e+00\n",
      "  -1.93997718e+00 -6.42680347e-01 -2.87187972e+00 -8.78345065e-01\n",
      "  -1.18116747e+00  1.80165228e+00 -5.23299089e-01  1.28516233e+00\n",
      "   5.65573883e-01  2.67294922e-01 -2.60108679e+00  1.01388340e+00\n",
      "   3.31308791e+00  3.90518669e+00  2.46622779e+00  1.92522877e+00\n",
      "   2.88450827e+00 -1.03440735e+00 -1.56330072e-01 -1.16111712e+00\n",
      "  -8.16844585e-01 -7.47398835e-01 -1.86405166e+00 -3.37178982e-01\n",
      "  -4.06618510e-01 -1.53657020e+00  7.69199355e-01 -2.00492569e+00\n",
      "  -2.04535620e+00 -4.38162857e-01 -5.12971494e-01 -2.04795419e+00\n",
      "  -2.31470438e-01 -1.40358092e+00  2.29953554e-01  1.67768365e+00\n",
      "   1.33759945e-01 -1.34220436e+00 -1.74585138e+00 -8.18639690e-01\n",
      "   4.39252267e-01 -3.97178083e+00 -1.29471629e+00 -1.26761622e+00\n",
      "   8.34530054e-01 -1.62955393e+00]]\n"
     ]
    }
   ],
   "source": [
    "print(paras[\"WRD_EMB\"].shape)\n",
    "print(paras[\"WRD_EMB\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.208546Z",
     "start_time": "2019-07-14T06:37:42.186422Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(13, 50)\n",
      "[[ 1.07483755e-01 -4.61282054e-02 -1.09460902e-02 -1.84773246e-01\n",
      "   1.78259483e-01  4.73607050e-02  4.81776362e-02 -1.44439323e-01\n",
      "   2.31756607e-01  2.38543082e-01 -2.54454081e-02 -5.00660086e-02\n",
      "   1.80203353e-01 -4.43203881e-01  1.31904331e-01  9.61572073e-02\n",
      "  -3.96034030e-02 -2.76891598e-01  5.14007097e-02 -2.46473798e-01\n",
      "  -1.35579735e-01 -2.66893268e-01  2.34378665e-01 -2.99851764e-01\n",
      "   5.54027386e-03  1.67338666e-01  1.12369116e-01  1.07232859e-01\n",
      "  -3.04131354e-02 -6.08423464e-02 -6.41920298e-02 -4.64976649e-02\n",
      "  -1.44549470e-01 -4.26620644e-02 -1.17600019e-01  2.09101316e-02\n",
      "  -3.15207180e-01  2.10171435e-02  1.04380235e-01  1.14968697e-01\n",
      "   2.99682830e-01  8.29542618e-02 -9.88975523e-02  3.88785950e-01\n",
      "  -4.21420293e-01  3.42267949e-02  2.08611693e-01  3.54049504e-01\n",
      "  -2.15470079e-02 -1.10387249e-01]\n",
      " [-5.36902252e-02 -1.15433197e-02 -7.02155264e-02 -9.52782341e-02\n",
      "  -5.37842624e-03  4.78770061e-02 -2.14661090e-03 -5.60695334e-02\n",
      "   1.97369266e-01  2.29959008e-01  8.49803488e-02  1.30940126e-01\n",
      "   8.89883604e-02 -1.87759937e-01 -5.82943408e-03  1.85145594e-02\n",
      "  -1.92183829e-02 -1.12638794e-01  7.53644869e-02 -1.86217204e-01\n",
      "  -2.62889936e-02 -2.16801765e-01  2.14973430e-01 -2.51043982e-01\n",
      "  -1.50327073e-01  7.39381675e-03  8.42255761e-02 -6.63289732e-02\n",
      "   5.90178413e-03 -3.40746145e-02 -1.02733131e-01 -3.98031356e-02\n",
      "  -1.71201881e-01 -7.45362075e-02 -1.14016762e-01 -6.51701380e-02\n",
      "  -2.44307888e-02  7.08684773e-02  5.73289969e-02  7.13818284e-02\n",
      "   5.45836378e-02 -5.29767474e-02 -7.78666505e-02  2.50104085e-01\n",
      "  -2.80186943e-01  1.40753616e-03  1.53212996e-01  1.45982193e-01\n",
      "  -9.19010603e-02 -1.33006776e-02]\n",
      " [-1.88849241e-01  9.97574597e-02 -1.54435622e-01  2.67420153e-01\n",
      "   1.36553888e-02 -8.82809331e-02 -3.19329999e-02  1.78156304e-01\n",
      "  -7.16890303e-02 -1.86556480e-01  1.54319457e-01  1.04096640e-01\n",
      "  -6.48203198e-02  4.99658630e-01 -4.50415015e-02 -2.03682992e-01\n",
      "   1.58023173e-02  2.28815534e-01  2.88855634e-02  2.49971255e-01\n",
      "   1.36606907e-01  1.46844021e-01 -1.46593338e-01  3.91199918e-01\n",
      "   1.64215892e-01 -2.75446592e-01 -1.20348466e-01 -3.08971195e-01\n",
      "   7.27935918e-02 -1.20193128e-02 -6.29000540e-02  3.39811420e-02\n",
      "   1.95353242e-01 -2.78282758e-03  6.74089437e-02  1.21252731e-01\n",
      "   2.41722056e-01  7.84020028e-02 -1.13209723e-01 -1.38390846e-01\n",
      "  -3.04250626e-01 -1.49438852e-01  3.22608111e-02 -4.90105733e-01\n",
      "   4.81860807e-01  4.07947075e-03 -1.50106613e-01 -2.68036901e-01\n",
      "   1.98200174e-01  1.88537731e-01]\n",
      " [-1.74831636e-01  9.18711571e-03 -1.48397627e-01  1.76013650e-01\n",
      "  -1.06171814e-03  8.57708540e-04 -9.58779186e-02  1.27219620e-01\n",
      "  -1.16459384e-01 -1.33207736e-01  7.73273564e-02  7.06590328e-02\n",
      "  -1.13507421e-01  4.00376683e-01 -7.42210861e-02 -1.55400570e-01\n",
      "   1.29714169e-02  2.43851083e-01  4.65031586e-02  1.92157460e-01\n",
      "   8.68921139e-02  1.45225399e-01 -1.47544226e-01  2.89995420e-01\n",
      "   2.48559667e-02 -2.18418965e-01 -1.53336990e-01 -2.27937537e-01\n",
      "   7.98340566e-02 -1.44296915e-02 -5.08239661e-02  9.75640468e-03\n",
      "   1.60482176e-01 -1.10054370e-02  4.36081642e-03  9.56459619e-02\n",
      "   2.47964113e-01  6.79763548e-02 -9.56145211e-02 -8.29677269e-02\n",
      "  -2.77395538e-01 -1.75739305e-01  5.79722872e-03 -4.31027245e-01\n",
      "   3.51586015e-01 -2.36399076e-03 -1.47652405e-01 -2.10156771e-01\n",
      "   9.60415785e-02  1.52964368e-01]\n",
      " [-7.70674720e-02 -1.45320489e-01 -1.12770720e-01 -2.97174143e-01\n",
      "  -1.35537851e-01  1.65084897e-01 -3.91559902e-02  9.16088897e-02\n",
      "   1.86735033e-01  4.48286569e-01 -1.34867090e-02  1.84988855e-01\n",
      "   1.84710403e-01 -5.41739326e-01  1.46646406e-01  1.81176524e-01\n",
      "  -4.27686913e-02 -3.82946778e-01  9.04725399e-02 -4.89084546e-01\n",
      "  -1.34323845e-01 -3.35241162e-01  4.47671016e-01 -6.09002534e-01\n",
      "  -2.68677486e-01  6.22007236e-02  2.90689177e-01  1.63455587e-01\n",
      "  -4.56297035e-03  5.46015404e-02 -5.69761705e-02  2.45553166e-02\n",
      "  -3.24226229e-01 -6.17912799e-02 -3.07816122e-02 -3.46028844e-01\n",
      "   1.17660265e-02  1.06709196e-01 -3.58633788e-02  1.18492224e-01\n",
      "   7.77592400e-02 -8.14317933e-02  9.36619630e-04  6.88976179e-01\n",
      "  -8.89666125e-01 -1.04858177e-01  3.71396568e-01  2.39433127e-01\n",
      "  -1.82657652e-01  1.33317345e-01]\n",
      " [ 3.82941758e-02 -5.72683465e-02 -1.18944484e-01  1.35809489e-01\n",
      "   1.63943544e-01  1.00850212e-01 -1.81228944e-01 -1.95742101e-02\n",
      "  -1.35829461e-01 -1.78536380e-01 -8.33997126e-02 -1.57241304e-01\n",
      "  -2.94127359e-01  2.83121152e-01  3.47879690e-02  7.29952081e-03\n",
      "   1.77060750e-01  2.64222648e-01  1.51105305e-01 -2.76106849e-03\n",
      "   8.40121540e-02  1.22940085e-01 -1.28825433e-01  3.88147988e-01\n",
      "   2.44823268e-01 -1.60850095e-01 -2.41794439e-01 -8.32598490e-02\n",
      "   1.94522617e-01 -1.19140580e-01  1.05947505e-01 -1.88217096e-02\n",
      "   4.53389703e-01  6.84264062e-02  1.77362228e-02  3.70176377e-01\n",
      "  -1.11404349e-01 -7.15144528e-03  1.51202756e-02  8.05394151e-02\n",
      "  -1.02293648e-01 -1.32159064e-01  2.30861631e-02 -3.25645863e-01\n",
      "   3.13762315e-01  1.81543679e-01 -1.83033523e-01  5.40369744e-02\n",
      "   5.89424046e-02  1.04213964e-01]\n",
      " [ 1.87153608e-01 -1.85960683e-01  1.39445909e-01 -5.08084840e-01\n",
      "  -1.56039199e-01  1.47782507e-01  3.63748623e-03 -5.44955029e-02\n",
      "   1.79060152e-01  5.11399126e-01 -1.49893781e-01  5.85793819e-02\n",
      "   2.82002375e-01 -9.33710689e-01  2.09704641e-01  4.00240775e-01\n",
      "  -2.35219323e-02 -6.29346746e-01  7.34353374e-02 -6.79879316e-01\n",
      "  -1.88005433e-01 -4.10158866e-01  5.88498844e-01 -8.72922489e-01\n",
      "  -2.62132658e-01  3.24115408e-01  4.07357854e-01  4.78334405e-01\n",
      "  -1.40648327e-01  1.13153977e-02  1.21917452e-01  3.31180829e-02\n",
      "  -4.68418523e-01  2.71232202e-03  5.36053132e-03 -4.59492956e-01\n",
      "  -2.60328719e-01 -3.25567874e-02  2.77819021e-02  1.48906235e-01\n",
      "   3.93865216e-01  2.14021890e-01  5.02591429e-02  1.14721663e+00\n",
      "  -1.20712274e+00 -3.71813352e-02  4.53221270e-01  4.30773906e-01\n",
      "  -2.84974836e-01 -8.62389920e-02]\n",
      " [-5.59201458e-02 -2.13777748e-02 -9.75948531e-02 -7.33676281e-03\n",
      "   1.20169700e-01 -4.39065472e-02 -7.83692494e-03  8.00915189e-02\n",
      "   7.96331967e-02  4.15801917e-02  8.30792256e-02  1.04490292e-01\n",
      "   1.17453984e-01  7.76847318e-02  5.52458225e-02 -5.33694823e-02\n",
      "   1.41408957e-02 -4.71571217e-02  1.26953339e-01 -3.31396972e-03\n",
      "   3.12186096e-02 -5.56669728e-02  9.94652351e-02  4.14825294e-03\n",
      "   1.39005760e-01 -1.06791023e-01  3.15404411e-02 -1.69548209e-01\n",
      "   4.02483871e-02 -6.95909701e-02 -5.93787610e-02 -1.86726935e-02\n",
      "  -4.78784505e-03 -3.09519398e-02  2.70240696e-02  1.07005553e-02\n",
      "  -6.88126798e-03  6.99752432e-02 -5.83044906e-02 -2.52381248e-02\n",
      "  -5.58063950e-02 -6.40059950e-02 -3.52691714e-02 -1.87709339e-02\n",
      "   9.60188058e-03 -4.47748691e-02  1.55038590e-02  3.98533561e-02\n",
      "   1.57305706e-01  1.21140793e-01]\n",
      " [ 2.94570321e-01 -1.95653706e-01  3.24844704e-01 -8.97008194e-02\n",
      "   1.98419846e-01  1.50939495e-01 -2.01099459e-01 -2.71018649e-01\n",
      "  -4.78788073e-01 -2.99374301e-01 -3.58521575e-01 -5.11098974e-01\n",
      "  -1.34362842e-01 -1.72579131e-01 -4.49319977e-02  1.86465330e-01\n",
      "  -7.59794283e-02 -5.75979862e-02 -4.88336060e-02  3.57191361e-02\n",
      "  -1.96482953e-01  3.64973815e-01 -2.41264569e-01  1.08219059e-01\n",
      "  -1.36135055e-01  3.92273889e-01 -3.38946128e-01  4.51995773e-01\n",
      "  -2.63007375e-01 -1.75494166e-01  3.07417422e-01  4.87465351e-02\n",
      "   1.38119854e-01  1.97433874e-01 -8.00683844e-02  1.55283167e-01\n",
      "  -1.27091136e-01 -2.42452185e-01  1.50103715e-03 -6.77180606e-02\n",
      "   3.88988788e-01  4.87093441e-01  6.25391060e-02 -9.43977898e-02\n",
      "   1.80172837e-01  4.02647723e-01 -3.04034561e-01  1.28196753e-01\n",
      "  -2.05820686e-01 -4.15469023e-01]\n",
      " [-1.03840493e-01  4.09940904e-03 -8.11931631e-02  1.05919549e-01\n",
      "  -2.71504307e-02 -2.26291341e-02 -3.73032119e-02  1.05183557e-01\n",
      "  -9.43196161e-02 -1.08580344e-01  3.42880644e-02  3.97424715e-02\n",
      "  -8.48727939e-02  2.74919581e-01 -6.46259476e-02 -9.49943778e-02\n",
      "   2.20794787e-02  1.83784365e-01  5.02669614e-03  1.63460969e-01\n",
      "   7.23638441e-02  1.21956449e-01 -1.32428316e-01  2.03014928e-01\n",
      "   3.64331243e-02 -1.46090616e-01 -8.62118599e-02 -1.21293601e-01\n",
      "   6.14627644e-02  2.94943132e-02 -2.42355931e-02  4.49998847e-03\n",
      "   9.97523074e-02  5.44000816e-03  4.38368612e-02  3.77603021e-02\n",
      "   1.64670991e-01  2.04907458e-02 -7.06570596e-02 -5.43119950e-02\n",
      "  -1.84615690e-01 -1.19555089e-01  1.80004500e-02 -2.92925292e-01\n",
      "   2.54162123e-01 -5.75601975e-02 -1.11130143e-01 -1.81228902e-01\n",
      "   8.09519960e-02  1.04386713e-01]\n",
      " [ 2.00365527e-01  4.71143631e-01  4.49426695e-01  1.61963587e-01\n",
      "  -3.52862708e-01 -4.57762421e-01  6.33924697e-01 -2.20958935e-01\n",
      "   2.38075229e-01 -2.61698024e-01  9.91544083e-02 -2.29913158e-02\n",
      "  -3.40440658e-02  6.82135765e-02 -2.97326900e-01 -1.27063183e-01\n",
      "  -1.33301248e-01  2.34690316e-01 -6.15964524e-01  6.11862092e-01\n",
      "   9.51025590e-02  5.52232486e-02 -4.41724483e-01  1.02060368e-01\n",
      "   5.65398115e-04  2.64207831e-01  1.99037299e-01  8.82144504e-02\n",
      "  -8.89109137e-02  4.05011788e-01 -6.79061513e-02 -8.08601261e-02\n",
      "  -2.69744421e-01 -7.59917932e-02  5.39252985e-02 -8.19744807e-02\n",
      "  -1.74974030e-01 -2.03503609e-01  2.76486703e-01 -1.14504241e-01\n",
      "   1.18790004e-01  1.97538226e-01 -1.03115618e-02 -6.65324157e-02\n",
      "   4.80472437e-01 -3.02477497e-01 -3.86369038e-03 -3.71170635e-01\n",
      "  -1.85017089e-02 -4.04827377e-01]\n",
      " [-1.85222496e-01  6.63123226e-02 -1.65979975e-01  2.13085834e-01\n",
      "   2.55386454e-02 -7.17291250e-02 -3.54963458e-02  1.60627840e-01\n",
      "  -1.08697942e-01 -1.91181538e-01  1.02832052e-01  9.75973465e-02\n",
      "  -1.02744801e-01  4.85855336e-01 -5.39472087e-02 -1.87552383e-01\n",
      "   3.62655690e-02  3.13537511e-01  4.11204972e-02  2.62155161e-01\n",
      "   1.46365544e-01  1.48565160e-01 -1.78041810e-01  3.71969123e-01\n",
      "   1.33270458e-01 -2.33510140e-01 -1.56611964e-01 -2.87723707e-01\n",
      "   1.07067266e-01  4.69729634e-03 -5.84329665e-02  4.17584487e-03\n",
      "   1.71723400e-01 -2.87955787e-02  4.90315151e-02  1.42150032e-01\n",
      "   2.06566054e-01  6.70533705e-02 -1.01989998e-01 -1.30104099e-01\n",
      "  -3.18935302e-01 -1.83587628e-01 -9.35993689e-04 -5.10304925e-01\n",
      "   4.83236020e-01 -5.11501891e-02 -1.59418401e-01 -2.55273230e-01\n",
      "   1.72501318e-01  1.70014905e-01]\n",
      " [-6.26579432e-03  4.56660836e-04  3.11457535e-02  8.20112819e-02\n",
      "  -4.35453329e-02 -3.73412189e-02 -1.26809591e-02  6.96426004e-02\n",
      "  -8.30312579e-02 -1.01673878e-01  1.36629264e-02 -3.57104878e-02\n",
      "  -4.08869489e-02  1.65421872e-01 -5.42490692e-02 -5.24595721e-02\n",
      "  -2.89067187e-03  3.58513238e-02 -4.30700005e-02  1.36597830e-01\n",
      "   2.13496393e-02  1.41779409e-01 -1.15476868e-01  1.57381948e-01\n",
      "   6.34212058e-02 -9.81764247e-02 -1.32522304e-02 -1.30564130e-02\n",
      "  -5.98819676e-03  2.34594853e-02  2.65533573e-02  2.26857885e-02\n",
      "   9.97809644e-02  6.53609671e-02  7.27751860e-02 -1.81446492e-02\n",
      "   1.11654320e-01 -3.12041655e-02 -5.77915839e-02 -3.47323099e-02\n",
      "  -6.50735619e-02 -3.91652229e-03  6.88213148e-02 -1.61836083e-01\n",
      "   1.64421529e-01 -9.72530798e-03 -1.06227185e-01 -1.39353652e-01\n",
      "   8.17761640e-02  4.94854389e-02]]\n"
     ]
    }
   ],
   "source": [
    "print(paras[\"W\"].shape)\n",
    "print(paras[\"W\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.230209Z",
     "start_time": "2019-07-14T06:37:42.212042Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0  1  2  3  4  5  6  7  8  9 10 11 12]]\n",
      "\n",
      "[[ 4  8  1 11  7  0  5  1  6 11  1 11  1]\n",
      " [10  0  3  7  1 10  0  9  1  7  4  7  3]\n",
      " [ 6  7 11 12  6  4  6  3  7 12  5 12 11]\n",
      " [ 1 12  9  2  9  6  4  8  4  2  0  2  9]]\n"
     ]
    }
   ],
   "source": [
    "X_test = np.arange(vocab_size)\n",
    "X_test = np.expand_dims(X_test, axis=0)\n",
    "print(X_test)\n",
    "print()\n",
    "softmax_test, _ = forward_propagation(X_test, paras)\n",
    "top_sorted_inds = np.argsort(softmax_test, axis=0)[-4:,:]\n",
    "print(top_sorted_inds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.257232Z",
     "start_time": "2019-07-14T06:37:42.232312Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1, 12,  9,  2,  9,  6,  4,  8,  4,  2,  0,  2,  9],\n",
       "       [ 6,  7, 11, 12,  6,  4,  6,  3,  7, 12,  5, 12, 11],\n",
       "       [10,  0,  3,  7,  1, 10,  0,  9,  1,  7,  4,  7,  3],\n",
       "       [ 4,  8,  1, 11,  7,  0,  5,  1,  6, 11,  1, 11,  1]])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_sorted_inds[::-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.289480Z",
     "start_time": "2019-07-14T06:37:42.262319Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  6, 10,  4])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_sorted_inds[::-1, 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2019-07-14T06:37:42.316688Z",
     "start_time": "2019-07-14T06:37:42.294486Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "costs's top 4 neighbor words: ['beating', 'of', 'deduction', 'the']\n",
      "beating's top 4 neighbor words: ['market', 'stock', 'costs', 'investing']\n",
      "game's top 4 neighbor words: ['is', \"loser's\", 'a', 'beating']\n",
      "a's top 4 neighbor words: ['game', 'market', 'stock', \"loser's\"]\n",
      "the's top 4 neighbor words: ['is', 'of', 'beating', 'stock']\n",
      "after's top 4 neighbor words: ['of', 'the', 'deduction', 'costs']\n",
      "of's top 4 neighbor words: ['the', 'of', 'costs', 'after']\n",
      "stock's top 4 neighbor words: ['investing', 'a', 'is', 'beating']\n",
      "investing's top 4 neighbor words: ['the', 'stock', 'beating', 'of']\n",
      "is's top 4 neighbor words: ['game', 'market', 'stock', \"loser's\"]\n",
      "deduction's top 4 neighbor words: ['costs', 'after', 'the', 'beating']\n",
      "loser's's top 4 neighbor words: ['game', 'market', 'stock', \"loser's\"]\n",
      "market's top 4 neighbor words: ['is', \"loser's\", 'a', 'beating']\n"
     ]
    }
   ],
   "source": [
    "for input_ind in range(vocab_size):\n",
    "    input_word = id_to_word[input_ind]\n",
    "    output_words = [id_to_word[output_ind] for output_ind in top_sorted_inds[::-1, input_ind]]\n",
    "    print(\"{}'s top {} neighbor words: {}\".format(input_word, len(output_words), output_words))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}