Notebook 2-NLP.ipynb

{
 "cells": [
  {
   "source": [
    "## Notebook 2 - Natural Language Processing\n",
    "\n",
    "This notebook will contain only some NLP methods such as n-grams because the full notebook was getting too cumbersum.\n",
    "Data has been transformed in other notebook, we can just load it in here :)\n"
   ],
   "cell_type": "markdown",
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "import matplotlib.pyplot as plt\n",
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# index_col stops index being read as column\n",
    "df_turk = pd.read_csv('turk-Data-renamed.csv', index_col=0)\n",
    "df_lab = pd.read_csv('lab-Data-renamed.csv', index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>button</th>\n",
       "      <th>event_type</th>\n",
       "      <th>target</th>\n",
       "      <th>time</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "      <th>step</th>\n",
       "      <th>turkId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>mousedown</td>\n",
       "      <td>alloc-slider-1</td>\n",
       "      <td>0.000</td>\n",
       "      <td>477.0</td>\n",
       "      <td>405.0</td>\n",
       "      <td>1</td>\n",
       "      <td>A35YFAFWP33C70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>mouseup</td>\n",
       "      <td>alloc-slider-1</td>\n",
       "      <td>0.111</td>\n",
       "      <td>478.0</td>\n",
       "      <td>405.0</td>\n",
       "      <td>1</td>\n",
       "      <td>A35YFAFWP33C70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>click</td>\n",
       "      <td>alloc-slider-1</td>\n",
       "      <td>0.111</td>\n",
       "      <td>478.0</td>\n",
       "      <td>405.0</td>\n",
       "      <td>1</td>\n",
       "      <td>A35YFAFWP33C70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>mousedown</td>\n",
       "      <td>alloc-slider-1</td>\n",
       "      <td>1.516</td>\n",
       "      <td>479.0</td>\n",
       "      <td>405.0</td>\n",
       "      <td>1</td>\n",
       "      <td>A35YFAFWP33C70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>mousedirchange</td>\n",
       "      <td>alloc-slider-1</td>\n",
       "      <td>2.395</td>\n",
       "      <td>543.0</td>\n",
       "      <td>403.0</td>\n",
       "      <td>1</td>\n",
       "      <td>A35YFAFWP33C70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1293068</th>\n",
       "      <td>1</td>\n",
       "      <td>click</td>\n",
       "      <td>html</td>\n",
       "      <td>258.656</td>\n",
       "      <td>500.0</td>\n",
       "      <td>2198.0</td>\n",
       "      <td>1</td>\n",
       "      <td>A15781PHGW377Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1293069</th>\n",
       "      <td>1</td>\n",
       "      <td>click</td>\n",
       "      <td>html</td>\n",
       "      <td>258.656</td>\n",
       "      <td>500.0</td>\n",
       "      <td>2198.0</td>\n",
       "      <td>2</td>\n",
       "      <td>A15781PHGW377Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1293070</th>\n",
       "      <td>1</td>\n",
       "      <td>click</td>\n",
       "      <td>html</td>\n",
       "      <td>258.656</td>\n",
       "      <td>500.0</td>\n",
       "      <td>2198.0</td>\n",
       "      <td>3</td>\n",
       "      <td>A15781PHGW377Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1293071</th>\n",
       "      <td>1</td>\n",
       "      <td>click</td>\n",
       "      <td>html</td>\n",
       "      <td>258.656</td>\n",
       "      <td>500.0</td>\n",
       "      <td>2198.0</td>\n",
       "      <td>4</td>\n",
       "      <td>A15781PHGW377Y</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1293072</th>\n",
       "      <td>1</td>\n",
       "      <td>click</td>\n",
       "      <td>html</td>\n",
       "      <td>258.656</td>\n",
       "      <td>500.0</td>\n",
       "      <td>2198.0</td>\n",
       "      <td>5</td>\n",
       "      <td>A15781PHGW377Y</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>820673 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         button      event_type          target     time      x       y  step  \\\n",
       "0             1       mousedown  alloc-slider-1    0.000  477.0   405.0     1   \n",
       "1             1         mouseup  alloc-slider-1    0.111  478.0   405.0     1   \n",
       "2             1           click  alloc-slider-1    0.111  478.0   405.0     1   \n",
       "3             1       mousedown  alloc-slider-1    1.516  479.0   405.0     1   \n",
       "4             1  mousedirchange  alloc-slider-1    2.395  543.0   403.0     1   \n",
       "...         ...             ...             ...      ...    ...     ...   ...   \n",
       "1293068       1           click            html  258.656  500.0  2198.0     1   \n",
       "1293069       1           click            html  258.656  500.0  2198.0     2   \n",
       "1293070       1           click            html  258.656  500.0  2198.0     3   \n",
       "1293071       1           click            html  258.656  500.0  2198.0     4   \n",
       "1293072       1           click            html  258.656  500.0  2198.0     5   \n",
       "\n",
       "                 turkId  \n",
       "0        A35YFAFWP33C70  \n",
       "1        A35YFAFWP33C70  \n",
       "2        A35YFAFWP33C70  \n",
       "3        A35YFAFWP33C70  \n",
       "4        A35YFAFWP33C70  \n",
       "...                 ...  \n",
       "1293068  A15781PHGW377Y  \n",
       "1293069  A15781PHGW377Y  \n",
       "1293070  A15781PHGW377Y  \n",
       "1293071  A15781PHGW377Y  \n",
       "1293072  A15781PHGW377Y  \n",
       "\n",
       "[820673 rows x 8 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_turk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmQAAAE/CAYAAADosN8VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO3de5xVdb3/8dcHRwQENBDkMiISYgYJykVBBQuDCnQoMKbAnHKymx5PdhHNkm4nKquTeX7qOOZgkOCBBAKPYKhQogKTgKBJIyJMYIxcR0cu43x+f6zv4Gacy0Zm7zXMfj8fj/3Y6/Jda33Wd23Yn/l+v3stc3dEREREJD4t4g5AREREJNMpIRMRERGJmRIyERERkZgpIRMRERGJmRIyERERkZgpIRMRERGJmRIyEYmFmZ1uZsvNrNzMfhV3PI3NzH5iZm+Y2euNtD83s96NtK88M/tbU9mPiCghE2l0ZrbZzN42szcTXt3ijqsJug54A2jv7t+qudLMiszsJ+kP6/Dx33eyYWZnAN8CPuzuXRo3sqOOpWdI5rLijENE6qeETCQ1rnD3tgmvbTUL6AuSM4EXPUV3p465fs8Edrr7jqPdUJ8LkcykhEwkTRJaKq41sy3AE2H5RWa2wsz2mNlaM7ssYZuzzGxZ6NZ73MzuMrMZYd1lZlZa4xibzezyMN3CzKaY2StmttPMHjazDjViucbMtoSute8l7OcEM7s1bFtuZsVmdoaZ/U/N7kUz+7OZ/Wcd5zzMzFaZ2d7wPiwsLwKuAb4bWhAvr7HddcCkhPV/DsunJMT0opl9OmGbPDN72sx+Y2a7gKnhPH4Vzu9VM7s+sbXIzE4xs/vNbLuZ/St0M55gZucC9wBDw/H3hPKfCsctD+W/Xcs5Xw48DnQL2xaF5Vea2YZwnZ8Kx0i8bjeb2TrgrXqSsk+Z2aZwPr80sxYJ1/o2M3vNzHaY2YNmdkrYZnl43xPiGZpw3DvMbHeom0/WcUzCtf+TmZWFz9JddZT7rZltNbN94TNzacK6IWa2Oqz7t5n9OixvZWYzwn73hM/J6fVdn7Cut0X/NvaG+phdV/wixwV310svvRrxBWwGLq9leU/AgQeBk4HWQHdgJ/Apoj+QPh7mO4VtngF+DZwEDAfKgRlh3WVAaV3HBv4TeBbIDtvfCzxUI5b7Qhz9gQPAuWH9d4AXgHMAC+s7AkOAbUCLUO40oAI4vZbz7QDsBq4GsoDPhfmOYX0R8JN66vE964GrgG6hriYCbwFdw7o8oBK4IRyvNfBV4MVQBx8A/hLOOytsMy/Uy8lAZ2Al8JWE/f2txvG3A5eG6Q8AF9QR+xHXBugTYv04cCLwXaAEaJlw3dYAZwCt69inA0+Geu0BbATyw7ovhf31AtoCfwL+UONaZyXsKw84BHwZOAH4WriuVstxTwDWAr8J9dQKuKS2OgImh89JFlGX7etAq4TP8tVhui1wUZj+CvBnoE041kCibuyGrs9DwPfCZ+FwTHrpdby+Yg9AL72a2yt8ub4J7AmveWF59Rdjr4SyN1d/cSYsW0zUetSDKME4OWHdH0k+IXsJGJmwrmv4Es5KiCU7Yf1KIDdMvwzk1HF+LwEfD9PXA4/WUe5qYGWNZc8AeWG6iKNMyGops6Y6zpAcbKmx/onqL/Awf3k47yzgdKIktHXC+s8BTybsr2ZCtiUkEO0biOuIawN8H3g4Yb4F8C/gsoTr9qUG9unAJxLmvw4sDdNLga8nrDunlmtdMyErSZhvE8p0qeW4Q4GyxO1r7Odv9cS8G+gfppcDPwROq1HmS8AK4Lwayxu6Pg8CBYmfYb30Op5f6rIUSY1x7n5qeI2rsW5rwvSZwFWhq2ZP6Bq7hCh56gbsdve3Esq/dhQxnAk8krDfl4B3iL7oqiX+ArCCqOUCopaaV+rY73SilhDC+x/qKNetlnhfI2oVfF/M7AtmtibhnPoRtdJV21pjk241ltWs+xOB7Qn7u5eoJaYu44laM18L3WVD6ylbM47DdeHuVSGWxLqoGXttEsu8Fvb7nv2H6eqksy6Hr727V4TJtrWUOwN4zd0rGwrOzL5lZi+FbsQ9wCm8e32uJWop/Efolhwblv+B6I+QWWa2zcx+YWYn0vD1+S5R6+3K0BX8pYbiE2nKNHhUJP0SB7FvJWoh+3LNQmZ2JvABMzs5ISnrkbD9W0QtG9XlTwA61dj3l9z96Vr23bOBGLcCHwTW17JuBrDezPoD5xJ1K9VmG9GXaqIewGMNHLvaEYP9Q33cB4wEnnH3d8xsDdGXcq3bEHUxZifMn5EwvZWoBea0OpKN9/zYwN1XATkhYbgeeLjGPuuyDfhIwrlY2O5f9R2vFmcAG8J0j7Df6v0n1nV16+q/OYYEONgK9DCzrPqSsjBe7Gai67PB3avMbDfh+rj7P4HPhXFvnwHmmFnH8Nn+IfDD8Ll8lKiF9lHquT7u/jpRlytmdgnwFzNb7u4lx3i+IrFQC5lIvGYAV5jZ6DCYvJVFg/Wz3f01YDXRF1XL8KVzRcK2G4FWZjYmJAi3EY0Vq3YP8NOQyGBmncwsJ8m4CoEfm9nZFjnPzDoCuHspsIqoZWOuu79dxz4eBfqY2efNLMvMJgIfBhYmGcO/icZEVTuZKGkpC+fzRaIWsvo8DNxoZt3N7FSihIFwHtuBJcCvzKx9GBj/QTMbkXD8bDNrGY7X0swmmdkp7n4I2EfU4piMh4ExZjYyXKtvESUbK5Lcvtp3zOwDFt1W40ageiD7Q8A3LfoRSFvgv4DZIZEpA6o4si6PxkqixHaamZ0cPqMX11KuHVESWAZkmdkPgPbVK81sspl1Cq2De8Lid8zso2b2kfAHxT6irtZ3Gro+ZnaVmVUn27uJPhvJXg+RJkcJmUiM3H0rkAPcSvRFtpVoQH31v83PAxcCu4DbicbNVG+7l2gcUSFRS8tbQOKvLn8LLACWmFk50QD/C5MM7ddEScQSoi/J+4kGyVebTtTiU1d3Je6+ExhLlHzsJOpiGuvubyQZw/3Ah0N31Tx3fxH4FdE4tH+H47+n9a+G+8I5rAOeJ0oSK3n3i/sLQEuigf+7gTlE3cUQjT/bALxuZtUxXw1sNrN9RD8YqO66rZe7vxzK/o7o3mtXEN0a5WAy2yeYDxQTjZ1bRFRHAL8nuhbLgVeB/UQ/bqjujvwp8HSoy4uO5oDu/k6ItzfRGLpSoh9U1LQY+D+iPxReCzEkdrF+AthgZm8SfTZz3X0/0IWo3vcRdasvI/pDBeq/PoOB58L+FgA3uvurAKELc9LRnKdI3Mw9JbcAEpEUMLOpQG93TyoRSGEcw4m+NHuGFo/jgkW3drjH3Wt2pYqIxEotZCJyVEKX241AYVNPxsystUX3Dssys+5ErYyPxB2XiEhNSshEJGkW3cx0D1G30X/HHE4yjGjA+G6iLsuXgB/EGpGISC3UZSkiIiISM7WQiYiIiMRMCZmIiIhIzI7rG8Oedtpp3rNnz7jDEBEREWlQcXHxG+7eqbZ1x3VC1rNnT1avXh13GCIiIiINMrM6H3+nLksRERGRmCkhExEREYmZEjIRERGRmB3XY8hERESkcRw6dIjS0lL2798fdyjHvVatWpGdnc2JJ56Y9DZKyERERITS0lLatWtHz549MbO4wzluuTs7d+6ktLSUs846K+nt1GUpIiIi7N+/n44dOyoZO0ZmRseOHY+6pVEJmYiIiAAoGWsk76celZCJiIhI7Hbu3MmAAQMYMGAAXbp0oXv37ofnDx48mNQ+Jk+ezLx585I+ZklJCQMGDKi3zKZNm5g1a1bS+3y/UjqGzMxOBQqBfoADXwJeBmYDPYHNwGfdfbdF6eRvgU8BFUCeu/89lfGJiIhI7XpOWdSo+9s8bUy96zt27MiaNWsAmDp1Km3btuXb3/520vuvrKw8pvjqUp2Q5ebmpmT/1VLdQvZb4DF3/xDQH3gJmAIsdfezgaVhHuCTwNnhdR1wd4pjExERkSauZivWtGnT+MlPfgLAJZdcwve+9z2GDx/OXXfddcR2t9xyC9deey1VVVVHLF+1ahXnnXceQ4cO5Z577jm8/JVXXuHSSy/l/PPPZ+DAgTz33HMATJkyhSeffJIBAwZw55131lnuWKWshczM2gPDgTwAdz8IHDSzHOCyUGw68BRwM5ADPOjuDjxrZqeaWVd3356qGEVEROT4tm/fPpYvXw5EXZYAN910EwcOHKCwsPA947ny8vIoKCjg4osv5pvf/Obh5V27duXxxx+nVatW/OMf/+Caa67hueeeY9q0adx1112Hu0IrKipqLXesUtll2QsoAx4ws/5AMXAjcHp1kuXu282scyjfHdiasH1pWJZZCdnUU9J4rL3pO5aIiEgK1OxKvP322xk2bBh33/3ejrY33niDt99+m4svvhiAq6++mieffBKAAwcOcP3117N27VqysrJ45ZVXaj1esuWOVioTsizgAuAGd3/OzH7Lu92TtantJwn+nkJm1xF1adKlSxcKCgoAGDJkCJ06dWLRoqjPu0ePHowaNYrCwkIAWrZsSV5eHvPmzWPHjh0AjB8/npKSEtauXQvAsGHDaNeuHYsXLwagV69eDB8+nKKiIgDatGnD5MmTmTNnDrt27QJg4sSJrF+/ng0bNgAwYsQIsrKyWLp0KQB9+vRh8ODBzJw5E4D27duTm5vLrFmz2LdvHwCTJk1i1apVbNy4MTqxNCoqKjo8WDI/P58lS5awZcsWAMaMGUNZWRkrV64EYODAgWRnZzN//nwAunXrxtixYyksLKSqqooWLVqQn5/PwoUL2bZtGwA5OTmUlpZSXFwMNJ/rBDBy5EgqKytZtmwZAH379qVfv37Mnj0bgA4dOjBhwgRmzJhBRUUFEP1ltnz5cjZt2gTA6NGjKS8vZ8WKFQD079+f3r17M3fuXAA6d+7MuHHjdJ10nXSddJ1Sfp0uuOACAHbt2sU777xDYysrKwOiz0NVVRVvvvkmAK1bt6ZVq1bs3r0bgKysKDV56623KCsrY+/evVRVVbF3714OHjzIzp07adu2LRUVFRw6dIgDBw7w1ltv0bJlS/bv3895553Hc889x+7du48YV9apU6fD+yorK+PUU0/l7bffprKykrKyMu644w66devGb37zGw4dOsRZZ52Fu7Nnzx4OHDjAG2+8wWmnncZPf/pTOnbsyBNPPEHbtm059dRTD59b27ZtadGiBfv27aO8vJy//OUvR1yn+ljUQ9j4zKwL8Ky79wzzlxIlZL2By0LrWFfgKXc/x8zuDdMPhfIvV5er6xiDBg3y1atXpyT+2KiFTEREYvDSSy9x7rnnHp5P96D+RImD+vfv3092djb//Oc/ad26NZdeeik5OTncdtttXHLJJdx1112Hx5hNnjyZCRMmsH//fn73u9+xePFi2rZte8S++/btS2FhIUOHDuVb3/oWS5cuZc2aNdxwww307t2bG2+8kfvuu4+vf/3rHDp0iOeee45bb731cCJbV7maatYngJkVu/ug2s45ZYP63f11YKuZnRMWjQReBBYA14Rl1wDzw/QC4AsWuQjYq/FjIiIima1Vq1bceuutDB48mCuvvJIPf/jDDW6Tm5tLXl4eOTk577lB6wMPPMBXvvIVhg4dekSydv3111NYWMhFF13Ea6+9xkknnQTA+eefzzvvvEP//v2588476yx3rFLWQgZgZgOIbnvREtgEfJEoCXwY6AFsAa5y913hthd3AZ8guu3FF9293uYvtZAd67HUQiYiIpHaWnTk/TvaFrKU3ofM3dcAtR14ZC1lHfhGKuMRERERaYp0p34RERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmCkhExERkSah5k1c6zN16lTuuOOOo9p/MtvMmzePF1988aj22xhSetsLEREROU419n0xj5N7X86bN4+xY8cmdQPaxqQWMhEREWmy/vznP3PhhRdy/vnnc/nll/Pvf//78Lq1a9fysY99jLPPPpv77ruv1u1/+tOfcs4553D55Zfz8ssvH15+3333MXjwYPr378/48eOpqKhgxYoVLFiwgO985zsMGDCAV155pdZyqaCETERERJqsSy65hGeffZbnn3+e3NxcfvGLXxxet27dOhYtWsQzzzzDj370o8MPeK9WXFzMrFmzeP755/nTn/7EqlWrDq/7zGc+w6pVq1i7di3nnnsu999/P8OGDePKK6/kl7/8JWvWrOGDH/xgreVSQV2WIiIi0mSVlpYyceJEtm/fzsGDBznrrLMOr8vJyaF169a0bt2aj370o6xcuZJx48YdXv/Xv/6VT3/607Rp0waAK6+88vC69evXc9ttt7Fnzx7efPNNRo8eXevxky13rNRCJiIiIk3WDTfcwPXXX88LL7zAvffee8TDwqPHYFPnfF3LAPLy8rjrrrt44YUXuP3229/zEPKjLXeslJCJiIhIk7V37166d+8OwPTp049YN3/+fPbv38/OnTt56qmnGDx48BHrhw8fziOPPMLbb79NeXk5f/7znw+vKy8vp2vXrhw6dIiZM2ceXt6uXTvKy8sbLNfY1GUpIiIiTUJFRQXZ2dmH52+66SamTp3KVVddRffu3bnooot49dVXD68fMmQIY8aMYcuWLXz/+9+nW7duR+zvggsuYOLEiQwYMIAzzzyTSy+99PC6H//4x1x44YWceeaZfOQjHzmchOXm5vLlL3+ZO++8kzlz5tRZrrGZu6dkx+kwaNAgX716ddxhNK7G/plxvcc6Pn6CLCIiqffSSy9x7rnnxh1Gs1FbfZpZsbsPqq28uixFREREYqaETERERCRmSshEREREYqaETERERAA4nseVNyXvpx6VkImIiAitWrVi586dSsqOkbuzc+dOWrVqdVTb6bYXIiIiQnZ2NqWlpZSVlcUdynGvVatWR9y+IxlKyERERIQTTzzxiMcSSXqpy1JEREQkZkrIRERERGKmhExEREQkZkrIRERERGKmhExEREQkZkrIRERERGKmhExEREQkZkrIRERERGKmhExEREQkZkrIRERERGKmhExEREQkZkrIRERERGKmhExEREQkZilNyMxss5m9YGZrzGx1WNbBzB43s3+G9w+E5WZmd5pZiZmtM7MLUhmbiIiISFORlYZjfNTd30iYnwIsdfdpZjYlzN8MfBI4O7wuBO4O7yIiIvXqOWVR2o61edqYtB1LMkccXZY5wPQwPR0Yl7D8QY88C5xqZl1jiE9EREQkrVKdkDmwxMyKzey6sOx0d98OEN47h+Xdga0J25aGZSIiIiLNWqq7LC92921m1hl43Mz+UU9Zq2WZv6dQlNhdB9ClSxcKCgoAGDJkCJ06dWLRoqjZukePHowaNYrCwkIAWrZsSV5eHvPmzWPHjh0AjB8/npKSEtauXQvAsGHDaNeuHYsXLwagV69eDB8+nKKiIgDatGnD5MmTmTNnDrt27QJg4sSJrF+/ng0bNgAwYsQIsrKyWLp0KQB9+vRh8ODBzJw5E4D27duTm5vLrFmz2LdvHwCTJk1i1apVbNy4keqsNV2Kioo4ePAgAPn5+SxZsoQtW7YAMGbMGMrKyli5ciUAAwcOJDs7m/nz5wPQrVs3xo4dS2FhIVVVVbRo0YL8/HwWLlzItm3bAMjJyaG0tJTi4mKg+VwngJEjR1JZWcmyZcsA6Nu3L/369WP27NkAdOjQgQkTJjBjxgwqKioAyMvLY/ny5WzatAmA0aNHU15ezooVKwDo378/vXv3Zu7cuQB07tyZcePG6TrpOuk6NXid0vv3u66T/j29n+tUH3N/T86TEmY2FXgT+DJwmbtvD12ST7n7OWZ2b5h+KJR/ubpcXfscNGiQr169Og3Rp9HUU9J4rL3pO5aISAppDJkcD8ys2N0H1bYuZS1kZnYy0MLdy8P0KOBHwALgGmBaeJ8fNlkAXG9ms4gG8++tLxkTkfdPX14iIk1LKrssTwceMbPq4/zR3R8zs1XAw2Z2LbAFuCqUfxT4FFACVABfTGFsIiIiIk1GyhIyd98E9K9l+U5gZC3LHfhGquIRERERaap0p34RERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmCkhExEREYmZEjIRERGRmDWYkJnZWcksExEREZH3JyuJMnOBC2osmwMMbPxwRESaj55TFqXtWJunjUnbsUSk8dWZkJnZh4C+wClm9pmEVe2BVqkOTERERCRT1NdCdg4wFjgVuCJheTnw5VQGJSIiIpJJ6kzI3H0+MN/Mhrr7M2mMSURERCSjJPMry51mttTM1gOY2XlmdluK4xIRERHJGMkkZPcBtwCHANx9HZCbyqBEREREMkkyv7Js4+4rzSxxWWWK4mly0vkrKYDN+rmEiIhIxkmmhewNM/sg4ABmNgHYntKoRERERDJIMi1k3wAKgA+Z2b+AV4HJKY1KREREJIM0mJC5+ybgcjM7GWjh7uWpD0tEREQkczSYkJnZTTXmAfYCxe6+JkVxiYiIiGSMZMaQDQK+CnQPr+uAy4D7zOy7qQtNREREJDMkM4asI3CBu78JYGa3Ez3LcjhQDPwideGJiIiINH/JJGQ9gIMJ84eAM939bTM7kJqwREREpKlJ+62gpo1J6/HilExC9kfgWTObH+avAB4Kg/xfTFlkIiIiIhkimV9Z/tjMHgUuAQz4qruvDqsnpTI4ERERkUxQb0JmZi2Ade7ej2i8mIiIiIg0snp/ZenuVcBaM+uRpnhEREREMk4yY8i6AhvMbCXwVvVCd78ymQOY2QnAauBf7j7WzM4CZgEdgL8DV7v7QTM7CXgQGAjsBCa6++ajORkRERGR41EyCdkPj/EYNwIvAe3D/M+B37j7LDO7B7gWuDu873b33maWG8pNPMZji4iIiDR5Dd4Y1t2X1fZKZudmlg2MAQrDvAEfI7qPGcB0YFyYzgnzhPUjQ3kRERGRZq3BhMzMLjKzVWb2ppkdNLN3zGxfkvv/b+C7QFWY7wjscffKMF9KdPd/wvtWgLB+bygvIiIi0qwl02V5F5AL/C/RY5S+AJzd0EZmNhbY4e7FZnZZ9eJainoS6xL3ex3R45vo0qULBQUFAAwZMoROnTqxaFF007oePXowatQoCgsLAWjZsiV5eXnMmzePHTt2ADB+/HhKSkpYu3YtAMOGDaNdu3YsXrwYgF69ejV0mse9oqIiDh6M7vubn5/PkiVL2LJlCwBjxoyhrKyMlStXAjBw4ECys7OZPz+6JV23bt0YO3YshYWFVFVV0aJFC/Lz81m4cCHbtm0DICcnh9LSUoqLox/ppuo6DR8+nKKiIgDatGnD5MmTmTNnDrt27QJg4sSJrF+/ng0bNgAwYsQIsrKyWLp0KQB9+vRh8ODBzJw5E4D27duTm5vLrFmz2Lcv+vtj0qRJrFq1io0bNwIwcuRIKisrWbYsajDu27cv/fr1Y/bs2QB06NCBCRMmMGPGDCoqKgDIy8tj+fLlbNq0CYDRo0dTXl7OihUrAOjfvz+9e/dm7ty5AHTu3Jlx48Y1+nWKhoamR/W/0Uy8TulUUFCQ4f+eqv+2T49M/X8v3fVc/f9Hc/l+qo+5vyfnObKA2Wp3H2Rm69z9vLBshbsPa2C7nwFXA5VAK6IxZI8Ao4Eu7l5pZkOBqe4+2swWh+lnzCwLeB3o5PUEOGjQIF+9enVdqxtF2u9K3Orz6TvY1L3pO5Y0Ken8XGfSnbZrUj2nj+o6PXSn/mNjZsXuPqi2dck8XLzCzFoCa8zsF2b2TeDkhjZy91vcPdvdexK1sD3h7pOAJ4EJodg1QPUTABaEecL6J+pLxkRERESai2QSsqtDueuJbntxBvCZYzjmzcBNZlZCNEbs/rD8fqBjWH4TMOUYjiEiIiJy3EhmDNk4d/8tsJ9wCwwzuxH4bbIHcfengKfC9CZgSC1l9gNXJbtPERERkeYimRaya2pZltfIcYiIiIhkrDpbyMzsc8DngbPMbEHCqnZEd9IXERERkUZQX5flCmA7cBrwq4Tl5cC6VAYlIiIikknqTMjc/TXgNWBo+sIRERERyTzJjCETERERkRRSQiYiIiISszoTMjNbGt5/nr5wRERERDJPfYP6u5rZCOBKM5tFjWdNuvvfUxqZiIiISIaoLyH7AdHd8rOBX9dY58DHUhWUiIiISCap71eWc4A5ZvZ9d/9xGmMSERERySgNPjrJ3X9sZlcCw8Oip9x9YWrDEhEREckcDSZkZvYzomdPzgyLbjSzi939lpRGJhmn55RFaTvW5mlj0nYsERGRhiTzcPExwAB3rwIws+nA84ASMhEREZFGkExCBnAqsCtMn5KiWESkOZqaxv8ypu5N37FERBpRMgnZz4DnzexJoltfDEetYyIiIiKNJplB/Q+Z2VPAYKKE7GZ3fz3VgYmIiIhkiqS6LN19O7AgxbGIiIiIZCQ9y1JEREQkZkrIRERERGLWYEJmZn9IZpmIiIiIvD/JtJD1TZwxsxOAgakJR0RERCTz1JmQmdktZlYOnGdm+8KrHNgBzE9bhCIiIiLNXJ0Jmbv/zN3bAb909/bh1c7dO+qxSSIiIiKNJ5n7kN1iZt2BMxPLu/vyVAYmIiIikimSebj4NCAXeBF4Jyx2QAmZiIiIpE4GPXotmRvDfho4x90PpDoYERERkUyUzK8sNwEnpjoQERERkUyVTAtZBbDGzJYCh1vJ3P0/UhaViIiISAZJJiFbgJ5jKSIiIpIyyfzKcno6AhERETkuZNBAc0mfZH5l+SrRryqP4O69UhKRiIiISIZJpstyUMJ0K+AqoENqwhERERHJPA3+ytLddya8/uXu/w18LA2xiYiIiGSEZLosL0iYbUHUYtYuZRGJiIiIZJhkuix/lTBdCWwGPtvQRmbWiuhu/ieF48xx99vN7CxgFlG359+Bq939oJmdBDwIDAR2AhPdfXPypyIiIiJyfErmV5YffZ/7PgB8zN3fNLMTgb+Z2f8BNwG/cfdZZnYPcC1wd3jf7e69zSwX+Dkw8X0eW0REROS40cM56D0AABUmSURBVOAYMjM7xcx+bWarw+tXZtbgb3498maYPTG8nGj82ZywfDowLkznhHnC+pFmZkdxLiIiIiLHpWQenfR7oJyom/KzwD7ggWR2bmYnmNkaYAfwOPAKsMfdK0ORUqB7mO4ObAUI6/cCHZM7DREREZHjVzJjyD7o7uMT5n8YkqwGufs7wAAzOxV4BDi3tmLhvbbWsPfc/8zMrgOuA+jSpQsFBQUADBkyhE6dOrFo0SIAevTowahRoygsLASgZcuW5OXlMW/ePHbs2AHA+PHjKSkpYe3atQAMGzaMdu3asXjxYgB69Wr+t1orKiri4MGDAOTn57NkyRK2bNkCwJgxYygrK2PlypUADBw4kOzsbObPnw9At27dGDt2LIWFhVRVVdGiRQvy8/NZuHAh27ZtAyAnJ4fS0lKKi4uBhq5Td9Ll5ZdfJisri6VLlwLQp08fBg8ezMyZMwFo3749ubm5zJo1i3379gEwadIkVq1axcaNGwEYOXIklZWVLFu2DIC+ffvSr18/Zs+eDUCHDh2YMGECM2bMoKKiAoC8vDyWL1/Opk2bABg9ejTl5eWsWLECgP79+9O7d2/mzp0LQOfOnRk3blyjXyfomqKajVdBQUGTuk7pVFBQ0Gj/7w0fPpyioiIA2rRpw+TJk5kzZw67du0CYOLEiaxfv54NGzYAMGLEiCbw7yl9/3+kU/V3XFO5Ts21ngHWrFmTkjwi8TrVx9zfk/McWcDsGeA77v63MH8xcIe7Dz2aEzWz24mei3kz0MXdK81sKDDV3Ueb2eIw/YyZZQGvA528ngAHDRrkq1evPpowjlrPKYtSuv+aNrf6fPoO1sTuAJ3Out48bUzajtUUpbWu9ZlOC32m9ZlOB30nHhszK3b3QbWtS6bL8mvA/5jZZjPbDNwFfDWJg3YKLWOYWWvgcuAl4ElgQih2DTA/TC8I84T1T9SXjImIiIg0F8n8ynIN0N/M2of5fUnuuysw3cxOIEr8Hnb3hWb2IjDLzH4CPA/cH8rfD/zBzEqAXUDu0Z2KiIiIyPEpmRvD/hfwC3ffE+Y/AHzL3W+rbzt3XwecX8vyTcCQWpbvJ3osk4iIiEhGSabL8pPVyRiAu+8GPpW6kEREREQySzIJ2QnhLvrA4fFgJ9VTXkRERESOQjK3vZgBLDWzB4huQ/El3r2Bq4iIiIgco2QG9f/CzNYR/UrSgB+7++KURyYiIiKSIZJpIcPdHwMeS3EsIukztcGnfzXisZrWfYSkmUrnZxr0uRZpZMmMIRMRERGRFFJCJiIiIhKzpBIyM2ttZuekOhgRERGRTNRgQmZmVwBrCGPIzGyAmS1IdWAiIiIimSKZFrKpRHfW3wOHH6XUM3UhiYiIiGSWZBKySnfXz2lEREREUiSZ216sN7PPE92x/2zgP4AVqQ1LREREJHMk00J2A9AXOAD8EdgL/GcqgxIRERHJJMm0kJ3j7t8DvpfqYEREREQyUTItZL82s3+Y2Y/NrG/KIxIRERHJMA0mZO7+UeAyoAwoMLMXzOy2VAcmIiIikimSujGsu7/u7ncCXyW6J9kPUhqViIiISAZJ5saw55rZVDNbD9xF9AvL7JRHJiIiIpIhkhnU/wDwEDDK3belOB4RERGRjNNgQubuF6UjEBEREZFMVWdCZmYPu/tnzewFwBNXAe7u56U8OhEREZEMUF8L2Y3hfWw6AhERERHJVHUO6nf37WHy6+7+WuIL+Hp6whMRERFp/pK57cXHa1n2ycYORERERCRT1TeG7GtELWG9zGxdwqp2wNOpDkxEREQkU9Q3huyPwP8BPwOmJCwvd/ddKY1KREREJIPUmZC5+15gL/A5ADPrDLQC2ppZW3ffkp4QRURERJq3ZO7Uf4WZ/RN4FVgGbCZqORMRERGRRpDMoP6fABcBG939LGAkGkMmIiIi0miSScgOuftOoIWZtXD3J4EBKY5LREREJGMk8yzLPWbWFlgOzDSzHUBlasMSERERyRzJtJDlAG8D3wQeA14BrkhlUCIiIiKZJJmHi7+VMDs9hbGIiIiIZKT6bgxbTi0PFefdh4u3T3FsIiIiIhmhvmdZtnP39gmvdonvDe3YzM4wsyfN7CUz22BmN4blHczscTP7Z3j/QFhuZnanmZWY2Tozu6DxTlNERESk6UpmDBlmdomZfTFMn2ZmZyWxWSXwLXc/l+i2Gd8wsw8T3fV/qbufDSzl3acAfBI4O7yuA+4+qjMREREROU4lc2PY24GbgVvCopbAjIa2c/ft7v73MF0OvAR0J/qRQPVYtOnAuDCdAzzokWeBU82s61Gci4iIiMhxKZkWsk8DVwJvAbj7NqIHjCfNzHoC5wPPAae7+/awr+1A51CsO7A1YbPSsExERESkWUvmPmQH3d3NzAHM7OSjOUC4h9lc4D/dfZ+Z1Vm0lmX+nkJm1xF1adKlSxcKCgoAGDJkCJ06dWLRokUA9OjRg1GjRlFYWAhAy5YtycvLY968eezYsQOA8ePHU1JSwtq1awEYNmwY7dq1Y/HixQD06tXraE71uFRUVMTBgwcByM/PZ8mSJWzZEj2mdMyYMZSVlbFy5UoABg4cSHZ2NvPnzwegW7dujB07lsLCQqqqqmjRogX5+fksXLiQbdu2AZCTk0NpaSnFxcVAQ9epeebfM2bMoKKiAoC8vDyWL1/Opk2bABg9ejTl5eWsWLECgP79+9O7d2/mzp0LQOfOnRk3blyjXydono3PBQUFTJo0iVWrVrFx40YARo4cSWVlJcuWLQOgb9++9OvXj9mzZwPQoUMHJkyYkJLr1Jw9/fTTbNiwAYARI0aQlZXF0qVLAejTpw+DBw9m5syZALRv357c3FxmzZrFvn37AFJwnZrn/x/V33G1fT8NHz6coqIiANq0acPkyZOZM2cOu3btAmDixImsX7++Ua9Tc61ngDVr1qQkj0i8TvUx9/fkPEcWMPs20biujwM/A74EPOTudza4c7MTgYXAYnf/dVj2MnCZu28PXZJPufs5ZnZvmH6oZrm69j9o0CBfvXp1gyd5LHpOWZTS/de0udXn03ewqXvTd6wkpLOuM7meQXWdLs22nkF1nS4ZXM/Q/OrazIrdfVBt6xrssnT3O4A5RK1c5wA/SDIZM+B+4KXqZCxYAFwTpq8B5ics/0L4teVFwN76kjERERGR5iKZLkvc/XHgcQAzO8HMJrn7zAY2uxi4GnjBzNaEZbcC04CHzexaYAtwVVj3KPApoASoAL54NCciIiIicryq78aw7YFvEHUYLyBKyL4BfAdYA9SbkLn736h9XBjAyFrKe9i/iIiISEapr4XsD8Bu4BkgnygRawnkuPuaerYTERERkaNQX0LWy90/AmBmhcAbQI9wTzERERERaST1Deo/VD3h7u8AryoZExEREWl89bWQ9TezfWHagNZhXg8XFxEREWlEdSZk7n5COgMRERERyVRJPVxcRERERFJHCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMRMCZmIiIhIzJSQiYiIiMQsZQmZmf3ezHaY2fqEZR3M7HEz+2d4/0BYbmZ2p5mVmNk6M7sgVXGJiIiINDWpbCErAj5RY9kUYKm7nw0sDfMAnwTODq/rgLtTGJeIiIhIk5KyhMzdlwO7aizOAaaH6enAuITlD3rkWeBUM+uaqthEREREmpJ0jyE73d23A4T3zmF5d2BrQrnSsExERESk2cuKO4DAalnmtRY0u46oW5MuXbpQUFAAwJAhQ+jUqROLFi0CoEePHowaNYrCwkIAWrZsSV5eHvPmzWPHjh0AjB8/npKSEtauXQvAsGHDaNeuHYsXLwagV69ejXeGTVRRUREHDx4EID8/nyVLlrBlyxYAxowZQ1lZGStXrgRg4MCBZGdnM3/+fAC6devG2LFjKSwspKqqihYtWpCfn8/ChQvZtm0bADk5OZSWllJcXAw0dJ2aZw4+Y8YMKioqAMjLy2P58uVs2rQJgNGjR1NeXs6KFSsA6N+/P71792bu3LkAdO7cmXHjxjX6dYLm2QBdUFDApEmTWLVqFRs3bgRg5MiRVFZWsmzZMgD69u1Lv379mD17NgAdOnRgwoQJKblOzdnTTz/Nhg0bABgxYgRZWVksXboUgD59+jB48GBmzpwJQPv27cnNzWXWrFns27cPIAXXqXn+/1H9HVfb99Pw4cMpKioCoE2bNkyePJk5c+awa1fUOTVx4kTWr1/fqNepudYzwJo1a1KSRyRep/qYe615T6Mws57AQnfvF+ZfBi5z9+2hS/Ipdz/HzO4N0w/VLFff/gcNGuSrV69OWfwAPacsSun+a9rc6vPpO9jUvek7VhLSWdeZXM+guk6XZlvPoLpOlwyuZ2h+dW1mxe4+qLZ16e6yXABcE6avAeYnLP9C+LXlRcDehpIxERERkeYiZV2WZvYQcBlwmpmVArcD04CHzexaYAtwVSj+KPApoASoAL6YqrhEREREmpqUJWTu/rk6Vo2spawD30hVLCIiIiJNme7ULyIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMVNCJiIiIhIzJWQiIiIiMWtSCZmZfcLMXjazEjObEnc8IiIiIunQZBIyMzsB+B/gk8CHgc+Z2YfjjUpEREQk9ZpMQgYMAUrcfZO7HwRmATkxxyQiIiKSck0pIesObE2YLw3LRERERJo1c/e4YwDAzK4CRrt7fpi/Ghji7jfUKHcdcF2YPQd4Oa2Bpt5pwBtxB5EBVM/po7pOD9Vz+qiu06e51fWZ7t6pthVZ6Y6kHqXAGQnz2cC2moXcvQAoSFdQ6WZmq919UNxxNHeq5/RRXaeH6jl9VNfpk0l13ZS6LFcBZ5vZWWbWEsgFFsQck4iIiEjKNZkWMnevNLPrgcXACcDv3X1DzGGJiIiIpFyTScgA3P1R4NG444hZs+2ObWJUz+mjuk4P1XP6qK7TJ2PquskM6hcRERHJVE1pDJmIiIhIRlJClgZm1tPM1teyPM/MujXGvpoTM9tsZqeF6TdTeJynzGxQmH7UzE6tpcxUM/v2MR7n+vA4MK8+r6agGdbzzPDotfVm9nszO/FY9teYmmFd329ma81snZnNMbO2x7K/xtLc6jlhX79L5fm8H82trs2syMxeNbM14TXgWPb3fighi1cecFQJmaSGu3/K3fcc637MrLZxmU8DlwOvHev+j3cprueZwIeAjwCtgfxjPc7xLMV1/U137+/u5wFbgOuP9TjHqxTXMyEZeU8SkolSXdfAd9x9QHitOdbjHC0lZOlzgpndZ2YbzGxJuPHtIGBmyMZbh784/svMnjGz1WZ2gZktNrNXzOyrcZ9AKpjZPDMrDvVyXT3lzMx+GVo/XjCziQnrvhuWrTWzabVse7KZLQrr1ydum1Am8a+974WWlr8Q3Xy4uswHzeyxEO9fzexDYXmRmf3azJ4Efl5z3+7+vLtvPsqqaVQZUs+PegCsJLqXYdplSF3vqz4HouQ37YORM6GeLXrG8y+B7x5d7TSuTKjrJsHd9UrxC+gJVAIDwvzDwGTgKWBQQrnNwNfC9G+AdUA7oBOwI2Ff6+M+p0asmw7hvTWwHugY6uG0sPzN8D4eeJzoliinE/1V3pXoYfQrgDaJ+6txjPHAfQnzp4T3w/VffUxgIPAC0AZoD5QA3w5llgJnh+kLgSfCdBGwEDihgXM9fF6q55TW84nA34FLVdepq2vgAeDfwJPVsaqeG7eegRuJWiMPn48+0ymr6yKiJ/+sI/r+PSnd9dykbnvRzL3q7zaBFhMlVrWpvhnuC0Bbdy8Hys1sv9XSd94M/IeZfTpMnwGcXUe5S4CH3P0d4N9mtgwYDIwAHnD3CgB331XLti8Ad5jZz4GF7v7XeuK5FHiken9mtiC8twWGAf8bNQoAcFLCdv8bYmuqMqme/x+wvIHjp1JG1LW7fzG04PwOmEiUoKVTs65ni8YXXwVcVs8x06VZ13VwC/A60JLoVhs3Az+qJ4ZGpy7L9DmQMP0Odd8DrrpcVY1tqurZ5rhkZpcRja0a6u79geeBVnUVr2f5Ed0lZnaGvTsw86vuvpF3/6L6mZn9oIHQaut+aQHs8XfHFwxw93MT1r+VcPzF4diFDRwnLTKpns3sdqIW5ZsaOHZKZFJdA4Qvt9lErRtpkyH1fD7QGygxs81AGzMraeD4jS5D6hp33+6RA0R/XAxp4PiNTglZvMqJuiQz1SnAbnevCP38F9VTdjkw0cxOMLNOwHCicUJLgC+ZWRsAM+vg7lsT/jHeE/7SrHD3GcAdwAUNHOfTFo3pawdcAYfHzLxqZleF45iZ9a9tB+4+Ohy7qQwqz4h6NrN8YDTwOXevSrJuGluzr+tQrnf1NmF//0iyfhpLs69nd1/k7l3cvae79wxx9E66hhpPs6/rULZr9TbAOKKu2bRqVi0ux6Ei4B4zexsYGnMscXgM+KqZrSPqu3+2nrKPENXRWqK/jL7r7q8Dj1n08+TVZnaQ6EkPt9bY9iPAL82sCjgEfK2ug7j7381sNrCG6FeRic3mk4C7zew2onFKs0I89TKz/yAalNsFWGdmj6Y5WcuIegbuCft6Jvo/lT+5e1q7HMiMujZgupm1D9Nr6zt+imRCPTcVmVLXM0MSaWG/af8hne7ULyIiIhIzdVmKiIiIxEwJmYiIiEjMlJCJiIiIxEwJmYiIiEjMlJCJiIiIxEwJmYiIiEjMlJCJiIiIxEwJmYiIiEjM/j84NhiPxWTHewAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x360 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Numbers of pairs of bars you want\n",
    "N = 6\n",
    "\n",
    "# Data on X-axis\n",
    "\n",
    "# Specify the values of blue bars (height)\n",
    "blue_bar = df_lab['target'].value_counts().divide( df_lab['turkId'].nunique() \n",
    "                                                 ).reindex(index = ['html','alloc-slider-1','alloc-slider-2','alloc-slider-3','alloc-slider-4','alloc-slider-5'])\n",
    "# Specify the values of orange bars (height)\n",
    "orange_bar = df_turk['target'].value_counts().divide( df_turk['turkId'].nunique() \n",
    "                                                    ).reindex(index = ['html','alloc-slider-1','alloc-slider-2','alloc-slider-3','alloc-slider-4','alloc-slider-5'])\n",
    "\n",
    "# Position of bars on x-axis\n",
    "ind = np.arange(N)\n",
    "\n",
    "# Figure size\n",
    "plt.figure(figsize=(10,5))\n",
    "\n",
    "# Width of a bar \n",
    "width = 0.3       \n",
    "\n",
    "# Plotting\n",
    "plt.bar(ind, blue_bar , width, label='Turk data')\n",
    "plt.bar(ind + width, orange_bar, width, label='Lab data')\n",
    "\n",
    "plt.ylabel('Relative count of target')\n",
    "plt.title('Frequency of targets for both classes.')\n",
    "\n",
    "# xticks()\n",
    "# First argument - A list of positions at which ticks should be placed\n",
    "# Second argument -  A list of labels to place at the given locations\n",
    "plt.xticks(ind + width / 2, ('html','alloc-slider-1','alloc-slider-2','alloc-slider-3','alloc-slider-4','alloc-slider-5'))\n",
    "\n",
    "ax = plt.gca()\n",
    "ax.set_axisbelow(True)\n",
    "ax.yaxis.grid(color='gray', linestyle='dashed')\n",
    "\n",
    "# Finding the best position for legends and putting it\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bag Of Words Models\n",
    "\n",
    "Can treat each target as a word as use NLP methods such as bagofwords.\n",
    "Will lose data regarding order but that can be analysed later with HMMs.\n",
    "\n",
    "'In this model, a text (such as a sentence or a document) is represented as the bag (multiset) of its words, disregarding grammar and even word order but keeping multiplicity. The bag-of-words model has also been used for computer vision. - https://en.wikipedia.org/wiki/Bag-of-words_model'\n",
    "\n",
    "disregarding grammer means nothing with this specific dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "# Here they suggest bayes filtering can be used to detect language, maybe I use too?\n",
    "https://stackoverflow.com/questions/101569/algorithm-to-detect-similar-documents-in-python-script\n",
    "\n",
    "# Naive Bayes\n",
    "Another idea from SPAM detection is using a naive bayes, here they get the count of each word for each email and bayes it.\n",
    "Could even use this to get the counts of 2/3-grams to compare?\n",
    "https://towardsdatascience.com/spam-filtering-using-naive-bayes-98a341224038\n",
    "\n",
    "TODO: I think for any of these to be used we would have to properly rename all the mouse items so they match"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Another idea from SPAM detection is using a naive bayes, here they get the count of each word for each email and bayes it.\n",
    "# Could even use this to get the counts of 2/3-grams to compare?\n",
    "# https://towardsdatascience.com/spam-filtering-using-naive-bayes-98a341224038\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import f1_score\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converts a pandas datafram of targets to a list of strings\n",
    "def targets_to_string(dataframe):\n",
    "    IDs = dataframe['turkId'].unique()\n",
    "    targets_text = []\n",
    "\n",
    "    for ID in IDs:\n",
    "        targets_series = dataframe[ dataframe['turkId'] == ID ] ['target']\n",
    "        targets =  ','.join(targets_series)\n",
    "        targets_text.append( targets)\n",
    "        #targets_text = targets\n",
    "\n",
    "    return targets_text\n",
    "\n",
    "turk_targets = targets_to_string(df_turk)\n",
    "lab_targets = targets_to_string(df_lab)\n",
    "turk_labels = np.zeros( len(turk_targets) )\n",
    "lab_labels = np.ones( len(lab_targets) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X = np.append(turk_targets, lab_targets)\n",
    "y = np.append(turk_labels, lab_labels)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Test train split\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 266,  282,  353,  314,  277,   80],\n",
       "       [  98,  198,  168,  134,  186,   74],\n",
       "       [  91,  145,   71,   91,  123,  105],\n",
       "       ...,\n",
       "       [ 811, 1127,  672,  871, 1011,  111],\n",
       "       [ 127,  224,  157,   77,  144,   60],\n",
       "       [ 366,  151,  127,  514,  186,   35]], dtype=int64)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Explore CountVecotrizer\n",
    "\n",
    "vectorizer = CountVectorizer(token_pattern='(.*?,)') \n",
    "counts = vectorizer.fit_transform(X_train)\n",
    "p = counts.toarray()\n",
    "p   \n",
    "# Normally with Bag-Of-Words models we end up with a spare matrix of mainly 0s. https://machinelearningmastery.com/gentle-introduction-bag-words-model/\n",
    "# However becuase our vocabulary is so limited (only 6 'words') the martix is full and so we dont have to wory about effeciency.\n",
    "# TF-IDF will not work due to small vocabulary.\n",
    "# Word hashing is used to help complexity but this is simple enough that we dont need it.\n",
    "\n",
    "# Limitations of bag of words are - need a carefuly designed vocabulary - Matrix can get very sparse - Context lost.\n",
    "# First 2 limitations dont apply to this data, last limitation is very relevant tho because we can imagine a user switching between sliders will have interesting order of targets that could be used in a model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['alloc-slider-1,', 'alloc-slider-2,', 'alloc-slider-3,', 'alloc-slider-4,', 'alloc-slider-5,', 'html,']\n"
     ]
    }
   ],
   "source": [
    "print(vectorizer.get_feature_names()) # Not sure if this worked correclty, lets try anyway lol."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Instantiating Multinomial Naive Bayes Classifier\n",
    "classifier = MultinomialNB()\n",
    "targets = y_train\n",
    "classifier.fit(counts, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 1., 1., 0., 0., 1., 1., 0., 1., 1., 0., 1., 0., 1., 1., 1., 0.,\n",
       "       0., 1., 0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
       "       1., 1., 0., 1., 0., 0., 0., 0., 1., 1., 1., 1., 1., 0., 1., 1., 0.,\n",
       "       1., 1., 1., 1., 1., 1., 1., 0., 0., 1., 0., 0., 0., 1., 1., 1., 0.,\n",
       "       0., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 1., 1., 1., 1., 1.,\n",
       "       0., 1., 0., 1., 0., 0., 1., 0., 0., 0., 0., 1., 1.])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_count = vectorizer.transform(X_test)\n",
    "predictions = classifier.predict(example_count)\n",
    "predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.29591836734693877\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[29, 66],\n",
       "       [ 3,  0]], dtype=int64)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "print( accuracy_score(y_test, predictions) )\n",
    "\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion_matrix(y_test, predictions)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.29591836734693877\n",
      "F1 score: 0.2283464566929134\n",
      "[[29 66]\n",
      " [ 3  0]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAEUCAYAAADJK5isAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAbOElEQVR4nO3de5RcVZ328e+ThBAIgQCBGEOAgAFlWAYBGS6KKA6gojDzyoggExheGfDVV2V0FGEEHVRmzVreBmacIJcIcokoC15gJmDWIJcBJIEghEACyCUSCAn3i5h0/94/9m6truquOtWp6lPd9XzWOqvr3PbZVUn9ap+999lbEYGZWaUxZWfAzDqPA4OZ1XBgMLMaDgxmVsOBwcxqODCYWQ0HhlFM0lmSLs2vt5f0qqSxZeeriMq8Fzj2YklntztP3cSBoY0kbSXpakmvSXpC0jFl5SUinoyIzSKip93XkjRX0sOSeiUd3+7rWes5MLTXecAfgKnAscC/S/qzcrM0LO4DPgPcU3ZGbGgcGNpE0kTgfwH/GBGvRsRtwLXAcU2mM0bSGbnEsVrSTyRtkfftKCkkzZH0pKQ1kk4fJJ2+Y8fl9Zsl/ZOk2yW9IulGSVMqjt9X0v9IelHSfZIOKprniDgvIhYCv2/mvdYj6WeSnpH0kqRbBgiwUyTdlN/LryTt0KprdyMHhvbZBeiJiOUV2+4D/gxA0nvyl26w5T35nOPz8n5gJ2Az4Nyqa70H2BU4GPi6pHcUzOMxwAnAtsB44Es5b9OB64Gzga3y9p9L2qbwux+EpGMavO/tBzn1P4FZOa/3AD+t2n8s8E/AFGDJAPutCePKzsAothnwUtW2l4BJALkEMblAOscC342IxwAknQY8IOmEimO+ERFvAPdJug+YDSwrkPZFfYFL0nzgY3n7p4AbIuKGvH6TpEXAh4F5BdIdVERcBlw2hPMu7Hst6SzgBUlbRETfZ3x9RNyS958OvCRpRkQ8tSH57VYuMbTPq8DmVds2B15pMp23Ak9UrD9BCuhTK7Y9U/H6dVJQKmKw83YAjqr8JSeVSqY1k/FWkTRW0jmSHpX0MvB43jWl4rA/BoCIeBV4nvTZ2RA4MLTPcmCcpFkV22YDSwEkvTc3Hw62vDef8zTpi9pne2A98Gwb8/4UcElETK5YJkbEORuasKRjG7zvgW4ljgGOAD4IbAHs2JdcxTEzKq6xGekW6OkNzW+3cmBok4h4DfgF8E1JEyUdQPrPfUnef2tuPhxsuTUndTnwRUkz83/4bwNXRsT6Nmb/UuCjkg7Nv9YTJB0kaTv4Yx+Dmwc7WdJ4SRNIX9yN8vlj8vv+aYP3/eQASU4C3gTWApuSPoNqH871NuNJdQ13+TZi6BwY2uszwCbAatIX/JSIWNpkGheSgsktwG9JNf2fa2Umq+Uv1BHA14DnSCWIL/On/y8zgNvrJHEj8AawPzA3vz5wA7L0E9It1O+AB4E7BzjmMuBM0i3EXqS6GRsieaAWa5akJcDBEbG27LxYezgwmFkN30qYWQ0HBjOr4cBgZjUcGMyshgODmdVwYDCzGg4MZlbDgcHMajgwmFkNBwYzq+HAYGY1HBjMrIYDg5nVcGAwsxoODGZWw4HBzGo4MJhZDQcGM6vhwGBmNRwYSiZpE0n/L8/J+LMNSOdYSTe2Mm9lyXNuPFx2PrqZB4MtKE9hfyrwdtJsUkuAb+Wp5jYk3eNIw8Hv3+a5IjqCpABmRcQjZefFBue5KwuQdCrwVeBkYAFpavvDSHMvbFBgIM0ytbwbgkIRksZ102dx6Psnxtrnewodu/g3by6IiMPanKUkIrzUWUhTor0KHFXnmI2B75OmRHs6v9447zsIWAn8PWnimVXACXnfN0hBZl2+xonAWcClFWnvCAQwLq8fDzxGKrX8Fji2YvttFeftD9xNmkj3blKJpG/fzaTZmm7P6dwITBnkvfXl/x8q8n8kaYLb5aQJXr5Wcfw+wB3Ai/nYc4Hxed8t+b28lt/vJyrS/wppLs1L+rblc3bO19gzr78VWAMcVPb/jVYse75z41i3audCC7BouPLlOobG9gMmAFfXOeZ0YF9gD9L8lPsAZ1TsfwspwEwnffnPk7RlRJzJn6ac2ywiLqiXEUkTgR8CH4qISaQv/5IBjtuKNI39D4Gtge8C10vauuKwY4ATSNPKjydNdT+Yt5A+g+nA14HzSTNi7wW8F/i6pJ3ysT3AF0kTzu4HHEyakYuI6JuNanZ+v1dWpL8VqfR0UuWFI+JRUtD4qaRNgYuAiyPi5jr5HUGCnugttAwnB4bGtgbWRP3i7bHANyNidUQ8RyoJHFexf13evy7S1PKvArsOMT+9wO6SNomIVTHwlHcfAVZExCURsT4iLgceAj5accxFEbE8It4A5pOC2mDWkepT1gFXkL70P4iIV/L1lwLvBIiIxRFxZ77u48B/AO8r8J7OjIg3c376iYjzgRXAXaQZt09vkN6IEUAvUWgZTg4Mja0FpkiqVx8z0FT1lVOwr60KLM1MVf9HkSbK/QSprmOVpOslvb1AfvryNL1i/Zkm8rM2IvpuhPu+uJWzbb/Rd76kXSRdJ+mZPGX9t+k/Xf1AnouI3zc45nxgd+BfI+LNBseOGEGwLnoKLcPJgaGxO0gTyR5Z55iBpqof6hTsr5FmdO7zlsqdEbEgIv6C9Mv5EOkL0yg/fXn63RDz1Ix/J+VrVkRsTpoYV/VPqf9zmGf5/j5wAXBWvlUaNVxiGIEi4iXSffV5ko6UtKmkjSSdIel5SY+Qfo3PkLSNpCn5+EuHeMklwIGStpe0BXBa3w5JUyV9LNc1vEm6JRnop+QGYBdJx0gaJ+kTwG7AdUPMUzMmAS8Dr+bSzClV+58Fdqo5q9aWklZLegD4AbA4Iv43qe7kR63McJkC6CEKLcPJgaGAiPguqQ/DGfxpWvjTSBVlu5GKyo8BvwHuB+4Bzh7itW4CrsxpLab/l3kMqXXjaVJN/fvIFXtVaawFDs/HriW1KBweEWuGkqcmfYlUsfkKqTRzZdX+s4B5kl6U9Nd10nmN1CQ8Kf89OW8/FdhT0qiZ5r4TSwzu4DQEkvYDzoqIQ/P6aQAR8Z1SMzbKSNoRuC4idi85K20ze/b4WHBDoyqYZNp2qxZHxN5tzhLgEsNQTSeVGvqspH/FnllhvQWX4eSej0MzUGWai17WtCih/qAIB4ahWQnMqFjfjqG3QlgXi4B1nRcXHBiG6G5glqSZpCbAo0kVbmZNEj0NW3OHn+sYhiB3Vvos6YGqZcD8QXog2hBJupzUh2RXSSslnVh2ntohgN4otgwnlxiGKHdtvqHsfIxWEfHJsvMwXDqxxODAYFai1MHJgcHMqvRG5wUG1zGYlaivxFBkKULSZElXSXpI0jJJ+0naStJNklbkv1s2SseBwaxEgVgXYwstBf0A+K+IeDtpbJBlpNHHFkbELGBhXq/LgWEDSDqp8VG2IUb7Z9zKEoOkzYEDSU+hEhF/iIgXSUMQzsuHzaP+k8KAA8OGGtX/aTvEKP+MRU+MKbSQxgVZVLFUfzY7kR7yu0jSvZJ+nJ/EnRoRqwDy320b5cqVj2YlSiM4Ff59XtPgIapxwJ7A5yLiLkk/oMBtw2AJdYyxEyfGRpNHzhgc47bYkgnTZ3Rgh9bB7T7lubKz0JTtp49j79kTRtRn/PhT61jzfE/hpoYWNleuJA2ie1dev4oUGJ6VNC0iVkmaRhrUt66OCgwbTd6KHf7u1LKzMar9+qR/KzsLo94+hz7V+KAsQn23CRssIp6R9JSkXSPiYdJAvA/mZQ5wTv57TaO0OiowmHWj3tZ2cPocaUTt8aTBg04g1SXOz93KnwSOapSIA4NZiQLxh2jd1zAilgAD1UMc3Ew6DgxmJWqy8nHYODCYlaynA7tEOzCYlSgQPS4xmFm13ha1SrSSA4NZiVKXaAcGM6vQ9xBVp3FgMCtRBC3r4NRKDgxmpVKrOzi1hAODWYkClxjMbACufDSzfgJ15JiPDgxmJXOJwcz6cXOlmdVIM1G5xGBmVTzhjJn1EyGXGMyslvsxmFk/aaAW30qYWT+tGwy2lRwYzEoU4OZKM+vPPR/NbEAeDNbM+knjMbjEYGZVfCthZv2kOobW3UpIehx4BegB1kfE3pK2Aq4EdgQeB/46Il6ol07n3dyYdZkeVGhpwvsjYo+KmbG/CiyMiFnAQgrMgO0Sg1mJArG+t+3NlUcAB+XX84Cbga/UO8ElBrOS9eZxHxstBQVwo6TFkk7K26ZGxCqA/HfbRom4xGBWoiZbJaZIWlSxPjci5lYdc0BEPC1pW+AmSQ8NJV8ODGYla6LycU1FvcGAIuLp/He1pKuBfYBnJU2LiFWSpgGrG13ItxJmJerr+VhkaUTSREmT+l4DhwAPANcCc/Jhc4BrGqXlEoNZyVr4dOVU4GpJkL7bl0XEf0m6G5gv6UTgSeCoRgk5MJiVKA3t1prAEBGPAbMH2L4WOLiZtBwYzMoUw9Jc2TQHBrMSeaAWMxuQn5Uws35aWcfQSm1trpR0mKSHJT0iqWH/bLNu1KrmylZqW4lB0ljgPOAvgJXA3ZKujYgH23VNs5GmG0dw2gd4JDehIOkK0sMcDgxmfQLWd9lgsNOBpyrWVwJ/3sbrmY04nVrH0M7AMNC7jZqD0hNgJwGM22LLNmbHrDN1W2BYCcyoWN8OeLr6oPx02FyACdNn1AQOs9GsU+sY2nlzczcwS9JMSeOBo0kPc5hZhQgVWoZT20oMEbFe0meBBcBY4MKIWNqu65mNVF3X8zEibgBuaOc1zEayiO6rYzCzhkRPb3c1V5pZAcNdf1CEA4NZibqxH4OZNRKpnqHTODCYlazrWiXMrL7AdQxmVqMzez46MJiVrLfXgcHMKkT4VsLMBuBbCTOr4eZKM6vRibcSnddJ26yLBMUeuW4meEgaK+leSdfl9ZmS7pK0QtKVeRiEuhwYzEoWBZcmfB5YVrH+z8D3ImIW8AJwYqMEHBjMyhQQvSq0FCFpO+AjwI/zuoAPAFflQ+YBRzZKx3UMZiVrcR3D94F/ACbl9a2BFyNifV5fSRqoua5BA4OkzeudGBEvF8unmdXTRKvEFEmLKtbn5jFTAZB0OLA6IhZLOqhv80CXbHSheiWGpTmByoT71gPYvlHiZlZfk89KrImIvevsPwD4mKQPAxOAzUkliMmSxuVSw4CDMlcbNDBExIzB9plZiwTQoluJiDgNOA0glxi+FBHHSvoZ8HHgCmAOcE2jtApVPko6WtLX8uvtJO01xLybWZWIYssG+ApwqqRHSHUOFzQ6oWHlo6RzgY2AA4FvA68DPwLevUFZNbOkDT0fI+Jm4Ob8+jHSlJGFFWmV2D8i9pR0b77I80U6SJhZEcWbIodTkcCwTtIYclyTtDXQ29ZcmXWLDn26skgdw3nAz4FtJH0DuI3Uk8rMWqENXR83VMMSQ0T8RNJi4IN501ER8UB7s2XWTTqvxFC05+NYYB0pbrkbtVkrdeBj1w2/5JJOBy4H3krqHHGZpNPanTGzrjESbyWATwF7RcTrAJK+BSwGvtPOjJl1hfwQVacpEhieqDpuHPBYe7Jj1oU68Fai3kNU3yNl+XVgqaQFef0QUsuEmbVCBzZX1isx9LU8LAWur9h+Z/uyY9Z9NJJKDBHRsD+1mW2gEioWiyjyrMTOwLeA3UiPcgIQEbu0MV9mXUIdeStRpE/CxcBFpF4YHwLmkx7fNLNW6MDmyiKBYdOIWAAQEY9GxBnA+9ubLbMu0ltwGUZFmivfzANKPirpZOB3wLbtzZZZl2jhQC2tVCQwfBHYDPi/pLqGLYC/bWemzLrJiGqV6BMRd+WXrwDHtTc7Zl1oJAUGSVdTJ8sR8VetzszGz/6emd9f2upkrcKhZ+1RdhZGveWxtuwsbLB6JYZzhy0XZl1sRN1KRMTC4cyIWdcaoZWPZtYuQUcOlOjAYFayTryVKDwak6SN25kRs641Ens+StpH0v3Airw+W9K/tj1nZt1iJAYG4IfA4cBagIi4D3eJNmsJRfFlOBUJDGMi4omqbT3tyIxZVwoVWxqQNEHSryXdJ2lpnu4BSTMl3SVphaQri0wYVSQwPCVpHyAkjZX0BWB5gfPMrIjW3Uq8CXwgImYDewCHSdqXNA/M9yJiFvACcGKjhIoEhlOAU0nT3j8L7Ju3mVkLqLfY0kgkr+bVjfISwAeAq/L2ecCRjdIq8qzEauDoxtkys6Y1V38wRdKiivW5ETG38gBJY0mjuL+NNIvco8CLEbE+H7ISmN7oQkVGcDqfAQoyEXFSo3PNrIDigWFNROxdN6mIHmAPSZOBq4F3DOWKRTo4/bLi9QTgL4GnCpxnZkW0ocUhIl6UdDPp1n+ypHG51LAd8HSj84vcSlxZuS7pEuCmoWXXzKq1qilS0jbAuhwUNiHNN/vPwH8DHycNyTgHuKZRWkPpEj0T2GEI55lZe00D5uV6hjHA/Ii4TtKDwBWSzgbuBRqOAF+kjuEF/lTYGQM8D3x1qDk3syotKjFExG+Adw2w/TFgn2bSqhsY8liPs0njPAL0RkQHPvJhNkJFsabI4Va3H0MOAldHRE9eHBTMWm2EPivxa0l7tj0nZl1IdOazEvXGfOxr3ngP8GlJjwKvkd5LRISDhVkrdGA5vF4dw6+BPSnQfdLMhqiE0kAR9QKDIM0+NUx5MetOIywwbCPp1MF2RsR325Afs67Tia0S9QLDWNIMVJ03hK3ZaDLCSgyrIuKbw5YTs25UQlNkEQ3rGMysvUZa5ePBw5YLs242kgJDRDw/nBkx61YjrcRgZsPBgcHMKpXR3bkIBwazsjkwmFk1lxjMrJYDg5nVcGAws35c+WhmA3JgMLNqI+3pSjMbBr6VMLP+RuDTlWY2HBwYzKxS3yjRnabI8PFm1k4tmldC0gxJ/y1pmaSlkj6ft28l6SZJK/LfLRul1bbAIOlCSaslPdCua5iNBoootBSwHvj7iHgHaZbr/yNpN9KUkgsjYhawkAJTTLazxHAxcFgb0zcb+fIUdUWWhklFrIqIe/LrV4BlwHTgCGBePmweBaaEaFtgiIhbSBPgmlk9xW8lpkhaVLGcNFiSknYkTXB7FzA1IlZBCh7Ato2yVHrlY35zJwFMGDOx5NyYDb8mKh/XRMTeDdOTNgN+DnwhIl5Oc1M3p/TKx4iYGxF7R8Te47VJ2dkxG34tnNRW0kakoPDTiPhF3vyspGl5/zRgdaN0Sg8MZl2t4IS2RUoVSkWDC4BlVRNCXQvMya/nANc0Sqv0Wwmzrte6fgwHAMcB90takrd9DTgHmC/pROBJ4KhGCbUtMEi6HDiIVGGyEjgzIi5o1/XMRqJWdnCKiNsYfD6YpqaDaFtgiIhPtitts9FEvZ3X9dG3EmZl8kNUZjYQj8dgZrVcYjCzap34dKUDg1mZAij2gNSwcmAwK5nrGMysn04dqMWBwaxMEb6VMLNaLjGYWS0HBjOr5hKDmfUXgJ+VMLNqbq40s1pulTCzaq5jMLP+/Ni1mVVLPR87LzI4MJiVzZWPZlbNJQYz6y/C/RjMrJZbJcyslm8lzKyfcM9HMxtIB5YYPHelWdlaO6nthZJWS3qgYttWkm6StCL/3bJROg4MZiVTRKGloIuBw6q2fRVYGBGzgIV5vS4HBrMyBdATxZYiyUXcAjxftfkIYF5+PQ84slE6rmMwK5FoqjQwRdKiivW5ETG3wHlTI2IVQESskrRtoxMcGMzKVjwwrImIvduZlT6+lTArW99I0Y2WoXtW0jSA/Hd1oxMcGMzKFKSHqIosQ3ctMCe/ngNc0+gE30qYlayVD1FJuhw4iFQfsRI4EzgHmC/pROBJ4KhG6TgwmJWthYEhIj45yK6Dm0nHgcGsTBHQ23l9oh0YzMrWeXHBgcGsbB6oxcxqOTCYWT+eiaqxl3vWrFnwwgVPlJ2PJkwB1pSdiVFuJH7GOxQ/dIM7L7VFRwWGiNim7Dw0Q9Ki4eqi2q264jN2YDCzfgLo6bxmCQcGs1IFhAPDaFPkkVfbMKP/M+7AWwk/RLUBGj0LL6lH0hJJD0j6maRNh3otSQdJui6//pikQUfhkTRZ0meGcI2zJH2p6PaqYy6W9PEmrrVj5fBjgyk43sDI1dcqUWQZRg4M7fVGROwREbsDfwBOrtyppOl/g4i4NiLOqXPIZKDpwGAlaf9j101zYBg+twJvy7+UyyT9G3APMEPSIZLukHRPLllsBiDpMEkPSboN+Ku+hCQdL+nc/HqqpKsl3ZeX/UlP0+2cSyv/ko/7sqS7Jf1G0jcq0jpd0sOSfgns2uhNSPp0Tuc+ST+vKgV9UNKtkpZLOjwfP1bSv1Rc++829IMcdRwYupOkccCHgPvzpl2Bn0TEu4DXgDOAD0bEnsAi4FRJE4DzgY8C7wXeMkjyPwR+FRGzgT2BpaTBPh/NpZUvSzoEmAXsA+wB7CXpQEl7AUcD7yIFnncXeDu/iIh35+stA06s2Lcj8D7gI8CP8ns4EXgpIt6d0/+0pJkFrtMdIqCnp9gyjFz52F6bSFqSX98KXAC8FXgiIu7M2/cFdgNulwQwHrgDeDvw24hYASDpUuCkAa7xAeBvACKiB3hpgOHBD8nLvXl9M1KgmARcHRGv52tcW+A97S7pbNLtymbAgop98yOiF1gh6bH8Hg4B3llR/7BFvvbyAtfqDh1Y+ejA0F5vRMQelRvyl/+1yk3ATdXP0Uvag8KzCTQk4DsR8R9V1/jCEK5xMXBkRNwn6XjSoCB9qtOKfO3PRURlAEHSjk1ed/TqwMDgW4ny3QkcIOltAJI2lbQL8BAwU9LO+bjBBuBYCJySzx0raXPgFVJpoM8C4G8r6i6m55GCbwH+UtImkiaRblsamQSskrQRcGzVvqMkjcl53gl4OF/7lHw8knaRNLHAdbpEwRaJYW6VcImhZBHxXP7lvVzSxnnzGRGxXNJJwPWS1gC3AbsPkMTngbl52K4e4JSIuEPS7bk58D9zPcM7gDtyieVV4FMRcY+kK4ElwBOk251G/hG4Kx9/P/0D0MPAr4CpwMkR8XtJPybVPdyjdPHnKDCvQdcIiA7s4KTowGKMWbfYYtw2sd/mxeLkghd+vHi4nhtxicGsbB344+zAYFamvubKDuPAYFay8GCwZtafB2oxs2oe2s3MBtSBzZXu4GRWogCiNwotReQH7x6W9Ei9R/MbcWAwK1PkEZyKLA1IGgucR3pgbzfgk5J2G0q2fCthVrJoXXPlPsAjEfEYgKQrgCOAB5tNyIHBrESv8MKCX8ZVUwoePkHSoor1uVUjXE0HnqpYXwn8+VDy5cBgVqKIOKyFyWmgSwwlIdcxmI0eK4EZFevbAU8PJSEHBrPR425glqSZksaTRucqMvhODd9KmI0SEbFe0mdJY2CMBS6MiKVDScuPXZtZDd9KmFkNBwYzq+HAYGY1HBjMrIYDg5nVcGAwsxoODGZW4/8DJpG5YTxHKkcAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "print( f'Accuracy score: {accuracy_score(y_test, predictions)}' )\n",
    "f1 = f1_score(y_test, predictions, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "\n",
    "# High accuracy because of imbalenced classes \n",
    "# https://machinelearningmastery.com/failure-of-accuracy-for-imbalanced-class-distributions/\n",
    "# This website has some nice references about imblaenced class distributions. TODO: use in diss.\n",
    "# data sampling – customized algorithms– cost sensitive algorithms– one class algorithms – threshold moving – probability calibration\n",
    "\n",
    "\n",
    "# https://machinelearningmastery.com/tour-of-evaluation-metrics-for-imbalanced-classification/\n",
    "# Better ways of dealing with imbalenced data\n",
    "\n",
    "cm = confusion_matrix(y_test, predictions)\n",
    "print(cm)\n",
    "\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of lab and turk data: 28\n",
      "Accuracy score: 0.2857142857142857\n",
      "F1 score: 0.2857142857142857\n",
      "[[1 5]\n",
      " [0 1]]\n"
     ]
    }
   ],
   "source": [
    "# Above has too much online data so there are uneven classes.\n",
    "# Look at only the first X (20 or so) online data for training\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "# TODO: make classes balenced\n",
    "\n",
    "lab_length = len(lab_targets)\n",
    "\n",
    "X_balanced = np.append( turk_targets[ :lab_length ] , lab_targets)\n",
    "\n",
    "y_balanced = np.append(turk_labels[ :lab_length ], lab_labels)\n",
    "\n",
    "print(f'Length of lab and turk data: {len(X_balanced)}')\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_balanced, y_balanced)\n",
    "\n",
    "vectorizer = CountVectorizer(token_pattern='(.*?,)') \n",
    "counts = vectorizer.fit_transform(X_train)\n",
    "\n",
    "classifier = MultinomialNB()\n",
    "targets = y_train\n",
    "classifier.fit(counts, targets)\n",
    "\n",
    "example_count = vectorizer.transform(X_test)\n",
    "predictions = classifier.predict(example_count)\n",
    "\n",
    "print( f'Accuracy score: {accuracy_score(y_test, predictions)}' )\n",
    "f1 = f1_score(y_test, predictions, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "cm = confusion_matrix(y_test, predictions)\n",
    "print(cm)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQAAAAEUCAYAAADENejrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAYLUlEQVR4nO3de7QdZZ3m8e+TBAiQcJEgasJNBLppliBEWsELXhpRUekL0yg4g800Ay4dW9QeELpFW0dn9VqO7UC3HQRRUAQvrMUITkB6kMsAkkRQIncUidBCADEBxHDOM39UHdgcztm79jm7Uufsej5r1eLUrrfqffcm+7ffS9X7yjYR0U5zmi5ARDQnASCixRIAIlosASCixRIAIlosASCixRIAhpik0ySdV/69k6T1kuY2Xa4qOsteIe05kj5dd5mGUQJAjSS9QNJFkh6XdK+k9zRVFtu/tL3A9kjdeUlaJul2SaOSjqk7v5i6BIB6nQH8HtgBOAr4F0l/1GyRNoqbgfcDq5ouSHSXAFATSVsCfw78ne31tq8BLgbe2+d15kg6taxBPCjpa5K2Lo/tIsmS/pOkX0paK+mUSa4zlnZeuX+lpH+QdK2kdZIuk7SoI/2rJP0/Sb+RdLOkg6uW2fYZtq8AftfPe+1G0rck/bukxyRdNUEgXSTp8vK9/FDSzoPKe5glANRnD2DE9h0dr90M/BGApNeUX67JtteU5xxTbm8AXgosAE4fl9drgD2BNwF/L+kPK5bxPcD7gBcCmwIfLcu2GLgE+DTwgvL170javvK7n4Sk9/R43ztNcur3gd3Lsq4Cvj7u+FHAPwCLgJsmOB4TmNd0AYbYAuCxca89BiwEKGsE21S4zlHA523fAyDpZOAWSe/rSPNJ208CN0u6GdgHuLXCtb8yFqAkXQi8s3z9aOBS25eW+5dLWgG8DfhqhetOyvY3gG9M4byzx/6WdBrwqKStbY99xpfYvqo8fgrwmKQdbd83nfIOu9QA6rMe2Grca1sB6/q8zkuAezv276UI3Dt0vPbvHX8/QRF8qpjsvJ2BIzp/mSlqGS/up+CDImmupM9JulvSb4FflIcWdSR75otuez3wCMVnF10kANTnDmCepN07XtsHWA0g6bXlsNxk22vLc+6n+EKO2Ql4Gvh1jWW/DzjX9jYd25a2PzfdC0s6qsf7nqgJ8B7gXcCbga2BXcYu15Fmx448FlA0Xe6fbnmHXQJATWw/DnwX+JSkLSUdRPGP+Nzy+NXlsNxk29Xlpc4HPixp1/If9n8HLrD9dI3FPw94h6S3lL++8yUdLGkJPDNGf+VkJ0vaVNJ8ii/oJuX5c8r3/fUe7/uXE1xyIfAU8DCwBcVnMN7byn6VTSn6Am5I9b+3BIB6vR/YHHiQ4ot8gu3VfV7jbIqgcRXwc4qe9Q8OspDjlV+cdwEfBx6iqBF8jGf/vewIXNvlEpcBTwIHAsvKv183jSJ9jaLp8yvgZ8D1E6T5BvAJiqr//hR9J9GDMiFI9EvSTcCbbD/cdFliehIAIlosTYCIFksAiGixBICIFksAiGixBICIFksAiGixBICIFksAiGixBICIFksAiGixBICIFksAiGixBICIFksAiGixBICIFksAiGixBICIFsu6ABFDRNIvKKaeHwGetr20W/oEgIjh8wbba6skTBOgYZI2l/S/yzXvvjWN6xwl6bJBlq0p5ZoJtzddjjZIAKioXNNuRbl4xQOSvt+xft90/AXFKj/b2T5iqhcp59s/ZADlqVW5QOnLuqUp10zYc2OVacgYuEzSSknH9UqcJkAFkk4ETgKOB5ZTLPl9KMXc+ddM8/I7A3fUvNDHrCFpXps+i7e8YUs//MhIpbQrf/LUap674vIy28vGJTvI9v2SXkixpuNtY2smTsh2ti4bxVJU64EjuqTZDPgCxVJU95d/b1YeOxhYA3yEYoGQB4D3lcc+SRFMNpR5HAucBpzXce1dKKL6vHL/GOAeio6enwNHdbx+Tcd5BwI3UixIeiNwYMexKylWz7m2vM5lwKJJ3ttY+f+2o/yHUywUegfFQhwf70h/AHAd8Jsy7enApuWxq8r38nj5fv+y4/r/jWKtwnPHXivP2a3MY79y/yXAWuDgpv9tDGLb7+WbecMDu1XagBV9/ts9DfhotzRpAvT2amA+cFGXNKcArwL2pVj/7wDg1I7jL6IIJIspvuRnSNrW9id4dqmvBbbP6lYQSVsCXwTeanshxZf8pgnSvYBiee8vAtsBnwcukbRdR7IJlwafxIsoPoPFwN8DZ1KsILw/8FqKJclfWqYdAT5MsXDnqymWLH8/gO2x1YH2Kd/vBR3XfwFFbeg51Vbbd1MEh69L2gL4CnCO7Su7lHcWMSMerbT1Ui5Bt3Dsb+AQ4JZu5yQA9LYdsNbdq6VHAZ+y/aDthyh+2d/bcXxDeXyDiyW31wNTbeOOAntL2tz2A554qbG3A3faPtf207bPB24D3tGR5iu273CxrPiFFMFrMhuAz9jeAHyT4sv9T7bXlfmvBl4OYHul7evLfH8B/Cvw+grv6RO2nyrL8xy2zwTuBG6gWKH4lB7XmzUMjOJKWwU7ANeUS8T/iGLJ9P/T7YT0AfT2MLCoR9t0oiW8O5emfnjcuf0s4f0M249L+kuKX+uzJF0LfMT2bT3KM1amxR37/Swp/rDtsYbq2Be0c3XiJ8fOl7QHRY1jKcVCnvOAld3eF/CQ7d/1SHMmcDFwnO2neqSdNYzZ4Gp9AD2vZd9DUQOtLDWA3q6j6Hg5vEuaiZbwnurS1I9TfHHGvKjzoO3ltv+E4pfwNoovRq/yjJXpV1MsUz/+haJcu9veimKBUXU/pfvPW7kq8heAs4DTyibO0BhgDaBvCQA92H6Mot17hqTDJW0haRNJp0p6RNJdFL+up0raXtKiMv15U8zyJuB1knaStDVw8tgBSTtIemfZvnuKoikx0c/HpcAe5dDlvLLWsBfwvSmWqR8Lgd8C6yX9AXDCuOO/Bl76vLOeb1tJD0q6BfgnYKXt/0zRt/GlQRa4SQZGcKWtDgkAFdj+PHAiRcfe2HLZJ1N0WO1F0Sa+B/gJ8FNgFfDpKeZ1OXBBea2VPPdLO4diNOF+ip7x11N2sI27xsPAYWXahyl68A9zxbvDpumjFB2M6yhqJxeMO34a8FVJv5H0H7pc53GKodaF5X+PL18/EdhP0tAs/91kDSCrA0+BpFcDp9l+S7l/MoDtzzZasCEjaRfge7b3brgotdlnn029/NJFldK+eMkDK93j3v5+pQYwNYspagFj1vDcDraIykYrbnXIKMDUTNSplapU9M01tu+rSACYmjXAjh37S5h6r3+0mA0bGvzpSACYmhuB3SXtSjG0diRFx1dEn8RIz1HS+qQPYArKm3o+QPFg0K3AhZPckRdTJOl8insw9pS0RtKxTZepDgZGXW2rQ2oAU1Te0ntp0+UYVrbf3XQZNpYmawAJABENKm4ESgCIaK1RJwBEtFJqABEtZsQGz20s/4wCTEOVOddieob9Mx6rAVTZ6pAAMD1D/Y9zhhjyz1iMeE6lrQ5pAkQ0qJgRqLnf4RkVADbZdEvPn79t08WobLP527BwqyWz6hmAPXfdGE8ED85Oi+exdJ/5s+oz/sV9G1j7yEjlOns6AUvz52/L0gM+0HQxhtoV53WddzQG4IC33Nc7UclWbdX7KmZUAIhoo9HUACLayYjfu7mvYQJARIPSCRjRciO5FTiinYwYSQ0gor1GMwoQ0U7FrcAJABGt1PTDQAkAEQ2yyY1AEe2l3AgU0VYmNYCIVksnYERLGWVOwIg2Sw0goqUyDBjRYsXKQKkBRLRWZgSKaClbA68BSJoLrAB+ZfuwbmkTACIaVsN9AB+iWLR2q14JMy14RIOKCUFUaatC0hLg7cCXq6RPDSCiUX1NCrpI0oqO/WW2l41L8wXgb4GFVS6YABDRIEM/w4BrbS+d7KCkw4AHba+UdHCVCyYARDRowHcCHgS8U9LbgPnAVpLOs330ZCekDyCiYaPMqbT1Yvtk20ts7wIcCfxbty8/pAYQ0ahiPoDcBxDRWnU8DGT7SuDKXukSACIaVPQB5FbgiNbKrcARLWXE06N5GjCitTInYERLZRQgouXSCRjRUpkTMKLl0gcQ0VLFlGAJABHt5AwDRrTW2IQgTUkAiGhYmgARLdV0H0CtA5CSDpV0u6S7JJ1UZ14Rs9WoVWmrQ201gHJq4jOAPwHWADdKutj2z+rKM2K2Geb7AA4A7rJ9D4CkbwLvAhIAIsYYnh7SOwEXA/d17K8B/rjG/CJmnab7AOoMABO9Kz8vkXQccBzAZvO3qbE4ETPTsAaANcCOHftLgPvHJyrnNV8GsHCrJc8LEBHDrOk+gDobHzcCu0vaVdKmFLOUXlxjfhGzkq1KWx1qqwHYflrSB4DlwFzgbNur68ovYrYa2jsBbV8KXFpnHhGzmT28fQAR0ZMYGR3OYcCIqKCu9n0VCQARDRrm+wAiohcX/QBNSQCIaNjQjgJERHcmfQARLTa8TwNGRAWjowkAEa1kpwkQ0WppAkS0WIYBI1osTYCIljL1PepbRQJARMOanAUnASCiSQYPaBhQ0nzgKmAziu/2t21/ots5CQARDRtgE+Ap4I2210vaBLhG0vdtXz/ZCZMGAElbdcvJ9m+nXs6IGDOoUQDbBtaXu5uUW9erd6sBrC5P7gxPY/sGdppySSMC6PtZgEWSVnTsLysn1X1GuSDPSuBlwBm2b+h2wUkDgO0dJzsWEQNioHoAWGt7adfL2SPAvpK2AS6StLftWyZLX2kuIklHSvp4+fcSSftXLXFEdGdX2/q7pn8DXAkc2i1dzwAg6XTgDcB7y5eeAL7UX3EiYlKuuPUgafvylx9JmwNvBm7rdk6VUYADbe8n6ccAth8p5/mPiGnTwIYBgRcDXy37AeYAF9r+XrcTqgSADZLmUMYgSdsBo9MtaURQTgk2mABg+yfAK/o5p0ofwBnAd4DtJX0SuAb4H/0XLyImNKAmwFT0rAHY/pqklRTtCYAjuvUqRkS/Zv6zAHOBDRRxqLlVDCKGUYMPA1QZBTgFOB94CcUKv9+QdHLdBYtojZncBACOBva3/QSApM9Q3Gn02XqKFNEiA3wYaCqqBIB7x6WbB9xTT3EiWmgmzggk6X9SFO0JYLWk5eX+IRQjARExCDN0QpCxnv7VwCUdr0/6aGFE9E8zsQZg+6yNWZCIVqqxg6+Knn0AknYDPgPsBcwfe932HjWWK6Il1GgToMqY/jnAVyjuVngrcCHwzRrLFNEuDQ4DVgkAW9heDmD7btunUjwdGBGDMFpxq0GVYcCnJAm4W9LxwK+AF9ZTnIiW6W9CkIGrEgA+DCwA/itFX8DWwF/VWaiINpmRowBjOuYUW8ezk4JExKDMxAAg6SK6FM32nw26MFr3BPP+beWgLxsd3nT0sU0XYejd/vPTmy5CZd1qALPnXUTMYjOyCWD7io1ZkIjWmuGdgBFRF9PoBHsJABENa7IJUHl2H0mb1VmQiNaayXcCSjpA0k+BO8v9fST9r3qKE9FCMzkAAF8EDgMeBrB9M7kVOGIg5OpbHar0AcyxfW9xN/AzRuopTkQLzfBRgPskHQC4XHHkg8Ad9RYrokVm4n0AHU6gaAbsBPwa+EH5WkQMgGbyMKDtB4EjN0JZItqnxvZ9FVVmBDqTCSopto+rpUQRbTOTAwBFlX/MfOBPgfvqKU5EC83kAGD7gs59SecCl9dWooiWmRV3AnbYFdh50AWJiI2vSh/AozxbSZkDPAKcVGehIlplpjYByrkA96GYBxBg1HaDxY0YMm52GLBrE6D8sl9ke6Tc8uWPGLQZ/izAjyTtV0/2Ee0mmn0WYNIAIGmsefAaiiBwu6RVkn4saVU9xYlooQHVACTtKOn/SrpV0mpJH+p1Trc+gB8B+wGH9846IqZksL/uTwMfsb1K0kJgpaTLbf9sshO6BQBBsRrQwIoXEc83oABg+wHggfLvdZJuBRYDUwoA20s6sUtmn59qQSPiWXWMAkjaBXgFcEO3dN0CwFyKFYGae1g5og2q1wAWSVrRsb/M9rLxiSQtAL4D/I3t33a7YLcA8IDtT1UuWkT0r78hvrW2l3ZLIGkTii//121/t9cFe/YBRES9BtUJWN64dxZwa9Umerf7AN40kFJFRHeDuxHoIIr1O98o6aZye1u3E7qtDPRIpSwjYloGVQOwfQ191tyzMEhE02bqw0ARUa86b/OtIgEgomkJABHtlRpARJslAES0WAJAREulEzCi5RIAItprRi8NFhH1ShMgoq1qnPCzigSAiKYlAES009iswE1JAIho2ixbG7ASSWdLelDSLXXlETEMZFfa6lBbAADOAQ6t8foRs1+5NFiVrQ61BQDbV1EsJBoR3TS4NFjjfQCSjgOOA5jPFg2XJmLja7ITsM4mQCW2l9leanvpJmzWdHEiNr421wAiWi0PA0W03DA2ASSdD1wH7ClpjaRj68orYrZqennw2moAtt9d17UjholGm6sCpAkQ0aQ8DBTRbpkPIKLNUgOIaK8MA0a0lYGaHvSpIgEgomHpA4hoqUwIEtFmdpoAEW2WGkBEmyUARLRXagARbWUgzwJEtFeTw4CNzwgU0XpjIwG9tgr6nY07ASCiYQOeD+Ac+piNOwEgoklV5wOsGAD6nY07fQARDSruBKz8875I0oqO/WW2l00n/wSAiKZV7wRca3vpILNOAIhoWF3LflWRABDRJLvR+wDSCRjRsEGOAvQ7G3dqABFNG2AToN/ZuBMAIprkTAgS0W7pBIxosTwNGNFeGQaMaCsDIwkAEa0knBpARKslAES0WAJAREuZfh4GGrgEgIiGpQ8gos0SACJayobR5toACQARTUsfQER7pQ8gos0SACJaKisDPWsdj679gb99b9Pl6MMiYG3ThejLFd9uugT9mn2fMexcPWmWB3+G7e2bLkM/JK0Y9Cyt8Vyt+IwTACJaysBIhgEjWsrgBIDZalqrskQlw/8ZN9gEyLTg09BrWSZJI5JuknSLpG9J2mKqeUk6WNL3yr/fKemkLmm3kfT+KeRxmqSPVn19XJpzJP1FH3ntUmUF2+kufTXjjY0CVNlqkABQrydt72t7b+D3wPGdB1Xo+/+B7Yttf65Lkm2AvgNANGSAy4P3KwFg47kaeFn5y3erpH8GVgE7SjpE0nWSVpU1hQUAkg6VdJuka4A/G7uQpGMknV7+vYOkiyTdXG4HAp8DditrH/9YpvuYpBsl/UTSJzuudYqk2yX9ANiz15uQ9NfldW6W9J1xtZo3S7pa0h2SDivTz5X0jx15/5fpfpBDJwFguEmaB7wV+Gn50p7A12y/AngcOBV4s+39gBXAiZLmA2cC7wBeC7xokst/Efih7X2A/YDVwEnA3WXt42OSDgF2Bw4A9gX2l/Q6SfsDRwKvoAgwr6zwdr5r+5VlfrcCnSvP7AK8Hng78KXyPRwLPGb7leX1/1rSrhXyaQcbRkaqbTVIJ2C9Npd0U/n31cBZwEuAe21fX77+KmAv4FpJAJtSLO30B8DPbd8JIOk84LgJ8ngj8B8BbI8Aj0nadlyaQ8rtx+X+AoqAsBC4yPYTZR4XV3hPe0v6NEUzYwGwvOPYhbZHgTsl3VO+h0OAl3f0D2xd5n1HhbzaIfcBDK0nbe/b+UL5JX+88yXg8vFLOknal8HNGC/gs7b/dVwefzOFPM4BDrd9s6RjgIM7jo2/lsu8P2i7M1AgaZc+8x1eGQVoteuBgyS9DEDSFpL2AG4DdpW0W5lusjXfrgBOKM+dK2krYB3Fr/uY5cBfdfQtLJb0QuAq4E8lbS5pIUVzo5eFwAOSNgGOGnfsCElzyjK/FLi9zPuEMj2S9pC0ZYV8WqLiCEBNowCpATTM9kPlL+n5kjYrXz7V9h2SjgMukbQWuAbYe4JLfAhYVq4COwKcYPs6SdeWw2zfL/sB/hC4rqyBrAeOtr1K0gXATcC9FM2UXv4OuKFM/1OeG2huB34I7AAcb/t3kr5M0TewSkXmDwGHV/t0WsDgBm8EkhusfkS03dbztvert6oWD5c/+uWVg34uIjWAiKalEzCipcaGARuSABDRMGdS0Ii2yoQgEe2VKcEiWq7BYcDcCBTRIAMedaWtivIBstsl3dXtkfExCQARTXI5I1CVrQdJc4EzKB482wt4t6S9up2TJkBEwzy4YcADgLts3wMg6ZvAu4CfTXZCAkBEg9bx6PIf+NuLKiafL2lFx/6ycTMmLQbu69hfA/xxtwsmAEQ0yPahA7ycJsqi2wnpA4gYHmuAHTv2lwD3dzshASBieNwI7C5pV0mbUsz21HWSlzQBIoaE7aclfYBiDoa5wNm2V3c7J48DR7RYmgARLZYAENFiCQARLZYAENFiCQARLZYAENFiCQARLfb/AYoYK/ghZFMmAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.42857142857142855\n",
      "F1 score: 0.4166666666666667\n",
      "[[2 4]\n",
      " [0 1]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQkAAAEUCAYAAAA4IMMhAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAckElEQVR4nO3de5hcVZnv8e8vFxIgEC5BLiEhCIEZZAwCIuANB0RABGaEIxcvMCgDDt51DgjDbfSoZ57HCw8cnSCIgGIQZZ4oYSLqQS4HkCQmSOQWECQSJQQIJEBIut/zx1odKk3Xrt3dVb2ru36f59kPVbVXrb2qSL+19lpr71cRgZlZPaOqboCZtTcHCTMr5CBhZoUcJMyskIOEmRVykDCzQg4SI5ikCyRdkx9PlbRK0uiq21VGbdtLlL1S0pda3aZO5SDRQpK2knSDpNWSHpd0YlVtiYg/RcSEiOhq9bEkzZT0oKRuSSe3+njWWg4SrXUp8AqwLXAS8G1Jb6i2SUNiEfBxYEHVDbHBc5BoEUmbAu8H/i0iVkXE7cBs4EP9rGeUpHNzT+QpSVdJmpj3TZMUkj4i6U+SnpZ0Tp16esqOyc9vkfTvku6Q9IKkX0iaVFN+f0n/T9JzkhZJOqhsmyPi0oj4FfByfz5rEUk/lvQXSSsl3dpHsJ0k6eb8WX4jaadmHbvTOUi0zm5AV0Q8VPPaIuANAJLelv8A621vy+85OW/vAl4PTAAu6XWstwG7AwcD50n625JtPBE4BXgdsBHw+dy2ycCNwJeArfLrP5G0TelPX4ekExt87ql13noTMD23dQHwg177TwL+HZgELOxjvw3QmKobMIJNAFb2em0lsBlA7llsUaKek4CvR8SjAJLOBu6TdEpNmQsj4iVgkaRFwAzg/hJ1f68niEm6Djgqv/5BYE5EzMnPb5Y0DzgC+H6JeuuKiB8CPxzA+67oeSzpAuBZSRMjouc7vjEibs37zwFWSpoSEU8Mpr3mnkQrrQI27/Xa5sAL/axnB+DxmuePk4L7tjWv/aXm8YukAFVGvfftBBxX+wtP6q1s35+GN4uk0ZK+KukRSc8Dj+Vdk2qKrQ8GEbEKeIb03dkgOUi0zkPAGEnTa16bASwGkPT2PCVZb3t7fs+TpD/aHlOBdcBfW9j2J4CrI2KLmm3TiPjqYCuWdFKDz93X6caJwNHAIcBEYFpPdTVlptQcYwLpNOnJwbbXHCRaJiJWAz8FLpK0qaS3kv6hX53335anJOttt+WqrgU+I2nn/I//fwGzImJdC5t/DfA+Se/Jv+LjJR0kaUdYv4bhlnpvlrSRpPGkP+Kx+f2j8uf+QYPP/ac+qtwMWAOsADYhfQe9HZHHeTYijU3c7VON5nCQaK2PAxsDT5H+2M+IiMX9rOMKUmC5FfgjacbgE81sZG/5j+to4IvAclLP4gu8+u9lCnBHQRW/AF4CDgRm5sfvGESTriKdZv0Z+ANwVx9lfgicTzrN2Ic0lmNNIN90xvpL0kLg4IhYUXVbrPUcJMyskE83zKyQg4SZFXKQMLNCDhJmVshBwswKOUiYWSEHCTMr5CBhZoUcJMyskIOEmRVykDCzQg4SZlbIQcLMCjlImFkhBwkzK+QgYWaFHCTMrJCDhNkIk29e/DtJP+9j3zhJsyQtkXS3pGmN6nOQMBt5PkX95EynAs9GxK7AN4CvNarMQaJikjaW9LOc4/LHg6jnJEm/aGbbqpJzkjxYdTuGo5z24L3Ad+sUOZpXs7BdDxwsSXXKAg4SpeUclvNyApllkm6qydc5GMeSsnFtHRHHDbSSnM/i0Ca0p6Vy0uJdi8rknCS7D1WbRphvAv8KdNfZP5mc7SznblkJbF1UoXOBliDps8BZwOnAXOAV4DBSVL59kNXvBDzU4mQ7w4akMZ30XbznXZvGime6SpWdf++axWyYqX1mRMzseSLpSOCpiJhfkAW+r15D8S3zI8JbwUZKK7cKOK6gzDhSBH8yb98ExuV9BwFLgc+RkvQsA07J+y4kBZy1+RinAhcA19TUPS3/TxyTn58MPErKKfpH4KSa12+ved+BwD2kX4p7gANr9t1CynJ1R67nF8CkOp+tp/3/WtP+Y0jJgx8iJcP5Yk35/YA7gedy2UuAjfK+W/NnWZ0/7wdq6v+fpNykV/e8lt+zSz7G3vn5DsDTwEFV/9toxrb3G8fF2mW7lNqAeQ3+rX4lf5eP5e/yxdp/S7nMXOCA/HhM/i5VVK9PNxo7ABgP3FBQ5hxgf2AvUr7P/YBza/ZvRwo2k0mB4FJJW0bE+byatm9CRFxe1BBJmwIXA4dHxGakQLCwj3JbATfmslsDXwdulFTbrTwROAV4HbAR8PmCQ29H+g4mA+cBl5Eyj+8DvB04T9Lrc9ku4DOkZL4HAAeTMpkRET1ZvGbkzzurpv6tSL2q02oPHBGPkALIDyRtAnwPuDIibilo7zASdEV3qa1hTRFnR8SOETENOB74dUR8sFex2cBH8uNjc5nCnoSDRGNbA09HcRf4JOCiiHgqIpaTeggfqtm/Nu9fGxFzSL+iAz3n7gb2lLRxRCyLvtMGvhd4OCKujoh1EXEt8ADwvpoy34uIhyLiJeA6UoCrZy3w5YhYC/yIFAC+FREv5OMvBt4IEBHzI+KufNzHgP8E3lniM50fEWtyezYQEZcBDwN3kzKbn9OgvmEjgG6i1DZQki6SdFR+ejmwtaQlQM9pdCGPSTS2ApjU4Fx5B1Kuyh6Ps2Ha+xW93vsiMKG/DYmI1ZI+QPrVv1zSHcDnIuKBBu3padPkmud/6Ud7VkREz4lzzx9xbVbzl3reL2k3Us9lX1Jy3zHA/KLPBSyPiJcblLmM9Ct4WkSsaVB22AiCtVFuTKJf9aae1i358Xk1r78M9GuA3D2Jxu4kDRYdU1DmSVJXucdUBp72fjXpj6vHdrU7I2JuRLyb9Iv6AOmPp1F7etr05wG2qT++TWrX9IjYnJR0uHCKjQYDZzmb+jdJv4IX5NOpEaPVPYnBcpBoICJWks7DL5V0jKRNJI2VdK6kZ3K37XHgXEnbSJqUy18zwEMuBN4haaqkicDZPTskbSvpqDw2sYZ02tLXz9AcYLc8bTsm9z72AF6zAq8FNgOeB1ZJ+hvgjF77/wq8/jXveq0tJT0l6T7gW8D8iPgoaazlO81scJUC6CJKbVVxkCghIr5OOn87F1hOmmc+mzTItgfpHP1R4F7g98AC4EsDPNbNwKxc13w2/MMeRZoleZI04v9O8qBgrzpWAEfmsitIMxNHRsTTA2lTP32eNCj6AqmXM6vX/guA70t6TtL/KKhnNWmaebP839Pz658F9pZ0UjMbXaV270k4q/gASDoAuCAi3pOfnw0QEV+ptGEjTL6u4OcRsWfFTWmZGTM2irlzJpUqu/2Oy+ZHxL4tbtJruCcxMOtXrWVL2XBQ0Ky07pJbVTy7MTD9X7Vm1oeoeLyhDAeJgVkKTKl5viMDn82wDhYBa9s7RjhIDNA9wHRJO5OmFY8nDdaZ9ZPoajhDXC2PSQxAXhh1Jmkd/P3AdXVWPtoASbqWtEZld0lLJZ1adZtaIYDuKLdVxT2JAcrLq+dU3Y6RKiJOqLoNQ6XdexIOEmYVSoupHCTMrEB3OEiYWR3uSZhZoUCsjdFVN6OQZzcGQdJpjUvZYIz077inJ1Fmq4qDxOCM6H/AbWKEf8eiK0aV2qri0w2zCqU7U7X3b3VbBYnREzaNMVsNn/uJjN5yS8ZNndLmi2o39HdbLq+6Cf0ydfIY9p0xflh9x489sZann+kqfX7ggct+GLPVVuzwuU9X3YwR7bcfGDH3a2lb+73nicaFsghVeipRRnu3zqwDdKNSWyOSxkv6raRFkhZLurCPMidLWi5pYd4+2qjetupJmHWaQLwSTfszXAP8fUSskjQWuF3STRFxV69ysyLizLKVOkiYVaiZA5c5f8aq/HRs3gY9nuPTDbOKdYVKbWVIGi1pISnb2s0RcXcfxd4v6V5J10ua0sf+DThImFUoEF2MKrWR8r/Mq9les4YkIroiYi/SjZD2k9T7/qA/A6ZFxBuBX/JqhvG6fLphVrHu8rMbT5e9EW5EPCfpFtKdxu+reX1FTbHLgK81qss9CbMKpWXZpXsShXLely3y442BQ0iJkmrLbF/z9CjSTZMKuSdhVqEmX+C1PSmnyWhSB+C6iPi5pItIGclnA5/MeUHXkXK3nNyoUgcJswpF0LTFVBFxL/CmPl6vzQV6NjVZ4cpwkDCrVLmFUlVykDCrUNC8nkSrOEiYVazMoGSVHCTMKhTI97g0s2LuSZhZXcPhHpcOEmYVShm83JMwswK+M5WZ1RUh9yTMrJjXSZhZXemmMz7dMLO62v9GuA4SZhUK8BSomdXnFZdm1pAzeJlZXel+Eu5JmFkBn26YWV1pTMKnG2ZWwMuyzayuQKzrbs4UqKTxwK3AONLf9vURcX6vMuOAq4B9gBXAByLisaJ627ufY9YBmpUwmFdzgc4A9gIOk7R/rzKnAs9GxK7AN3DeDbP21jO70Yw0f5E0ygV6NK9m7boeOFhSYeUOEmYV645RpTZKpPkrkQt0MvAEQESsA1YCWxe1z2MSZhXq54rLhmn+IqIL2Ctn8rpB0p4RcV9Nkb4OVph53D0Js4o1cUxivYh4DriFlAu01lJgCoCkMcBEUiavuhwkzCqUbl+nUlsjZXKBArOBj+THxwK/jojCnoRPN8yqFM2bAqVcLtDLgaslLSH1II5vVKmDhFmFmnnTmZK5QF8GjutPvQ4SZhXztRtmVlfPmEQ7a+nApaTDJD0oaYmks1p5LLPhqlkDl63Ssp5EHjy5FHg3adrlHkmzI+IPrTqm2XDT6Xem2g9YEhGPAkj6EWlJqIOEWY+AdR18qfj65Z/ZUuAtLTye2bAzHMYkWhkkSi3/zOvPTwMYveWWLWyOWXvq5CCxfvlntiPwZO9CETETmAkwbuqUwpVfZiPNcBiTaOXJ0D3AdEk7S9qItLJrdguPZzYsRajUVpWW9SQiYp2kM4G5wGjgiohY3KrjmQ1XHZ3mLyLmAHNaeQyz4Syis8ckzKwh0dXduVOgZlZCleMNZThImFWo09dJmFkjkcYl2pmDhFnFOnp2w8yKBR6TMLNC7b/i0kHCrGLd3e0dJNp7gtZshIto3rJsSVMk/V9J90taLOlTfZQ5SNJKSQvzdl5fddVyT8KsYk083VgHfC4iFkjaDJgv6eY+bvR0W0QcWbZSBwmzijVrCjQilgHL8uMXJN1Puq/LoG705NMNs4r143SjYS7QHpKmkW6v3zsXKMABkhZJuknSGxq1zz0JswoF/boMvGEuUABJE4CfAJ+OiOd77V4A7BQRqyQdAfwXML2oPvckzCoWJbcyJI0lBYgfRMRPX3OsiOcjYlV+PAcYK2lSUZ3uSZhVKSCaNAUqSaQ0fvdHxNfrlNkO+GtEhKT9SB2FFUX1OkiYVayJKy7fCnwI+L2khfm1LwJT03HiO6QkwWdIWge8BBw/4ITBkjYvemMf5zpmNgBNnN24nb5vQF1b5hLgkv7UW9STWEw6Fao9aM/zIEcnMxu4YX3tRkRMqbfPzJokgDYPEqVmNyQdL+mL+fGOkvZpbbPMOkdEua0qDYOEpEuAd5EGRABeBL7TykaZdZRmzoG2QJnZjQMjYm9JvwOIiGdyHg0zGzQ1bQq0VcoEibWSRpFjmaStge6WtsqsU0T7D1yWGZO4lLSCaxtJFwK3A19raavMOslwP92IiKskzQcOyS8dFxH3tbZZZp2kvXsSZVdcjgbWkuKZr/cwa6Y2v1t2mdmNc4BrgR1ImcF/KOnsVjfMrGMM99MN4IPAPhHxIoCkLwPzga+0smFmHaGJF3i1Spkg8XivcmOAR1vTHLMO1OanG0UXeH2D1PwXgcWS5ubnh5JmOMysGdp8CrSoJ9Ezg7EYuLHm9bta1xyzzqPh2pOIiMuHsiFmHaniQckyGo5JSNoF+DKwBzC+5/WI2K2F7TLrEGr7040yax6uBL5HWvFxOHAd8KMWtsmss7T5FGiZILFJRMwFiIhHIuJc0lWhZtYM3SW3ipSZAl2Tb7D5iKTTgT8Dr2tts8w6xAi56cxngAnAJ0k32vwY8E+tbJRZJ1GU2xrWUy4XqCRdLGmJpHsl7d2o3jIXePVkAHqBV288Y2bN0rzxhjK5QA8nJeOZDrwF+Hb+b11Fi6luoKD5EfGP/Wh8KeOeWM2un/EyjFZ6+63/XHUTRrwH/vStSo5bMhfo0cBV+Tb6d0naQtL2+b19KupJ9Ou222Y2MP1YTDVJ0rya5zMjYmafddbPBToZeKLm+dL8Wv+DRET8qri9ZtYUQ5sLtK+DDSw5j5kNgaCp05uNcoGSeg616TJ2BJ4sqtM3kDGrWBNnNxrmAgVmAx/Osxz7AyuLxiOgHz0JSeMiYk3Z8mZWUvNmN8rkAp0DHAEsIV3hfUqjSstcu7EfKTpNBKZKmgF8NCI+MYAPYWa9DW0u0AD+pT/1ljnduBg4kpyePCIW4WXZZk1R9lSjysvJy5xujIqIx9PpznpdLWqPWedp82XZZYLEE/mUIySNBj4BPNTaZpl1kOF+PwngDNIpx1Tgr8Av82tm1gRq83x4Za7deAo4fgjaYtZ5Kh5vKKPM7MZl9NEhiojTWtIis04z3IME6fSix3jgH9hw7beZDcZwDxIRMav2uaSrgZtb1iKzDtPupxsDWZa9M7BTsxtiZu2pzJjEs7zaIRoFPAOc1cpGmXWUNu9JFAaJfMHIDNJ9LQG687JOM2uGaP8p0MLTjRwQboiIrrw5QJg12wi4pf5vy9ws08z6TwzjazckjYmIdcDbgI9JegRYTfpcEREOHGbN0Ob986Ixid8CewPHDFFbzDrPMF9xKUhZu4aoLWadaRgHiW0kfbbezoLbY5lZP7T77EZRkBhNytzV3he7mw13w7gnsSwiLhqylph1ooqnN8somgJ1D8JsCDRzClTSFZKeknRfnf0HSVopaWHezmtUZ1FP4uByzTKzQWluT+JKUva9qwrK3BYRR5atsCiD1zPl22VmA9XMKdCIuDWn+GsaJ+cxq1r5ZdmTJM2r2QZ646cDJC2SdJOkNzQq7DR/ZhXq55LrUrlAG1gA7BQRqyQdAfwXML3oDe5JmFVtCC/wiojnI2JVfjwHGCtpUtF7HCTMKjaUF3hJ2i7fAqInO98ocuKteny6YVa1Jg5cSroWOIg0frEUOB8YC+tzgR4LnCFpHfAScHyjW0A4SJhVrbmzGyc02H8JaYq0NAcJsyoN86tAzWwoOEiYWZHhfBWomQ0Bn26YWX3D4CpQBwmzqjlImFk9PXfLbmcOEmZVa/Mg0bJl2Y1ufmFmiSJKbVVp5bUbVwKHtbB+s+Evp/krs1WlZUEiIm4lJRc2syJtnuav8jGJfOOM0wDGs0nFrTEbeu0+cFn5peIRMTMi9o2IfccyrurmmA099yTMrC5f4GVmDbV5kGjlFOi1wJ3A7pKWSjq1VccyG656FlMN1Z2pBqJlPYlGN78ws0Td7d2V8OmGWZV8gZeZNdLu95OofArUrOM1cQq0RC5QSbpY0hJJ90rau1GdDhJmFWvywOWVFF8OcTgpGc900iLGbzeq0EHCrEoBRJTbylTX+HKIo4GrIrkL2ELS9kV1ekzCrGL9GJOYJGlezfOZETGzn4ebDDxR83xpfm1ZvTc4SJhVqJ83nWlGLlD18ZqT85i1rX6cSjTJUmBKzfMdgSeL3uAxCbOKDfGKy9nAh/Msx/7Ayoioe6oB7kmYVW9oc4HOAY4AlgAvAqc0qtNBwqxizbwuo0Qu0AD+pT91OkiYVSkAX7thZkXafVm2g4RZ1Sq8E3YZDhJmFfOdqcysPl8qbmZF0orL9o4SDhJmVfPApZkVcU/CzOqL8DoJMyvm2Q0zK+bTDTOrK7zi0swacU/CzAq1d4xwkDCrmqdAzay+ALocJMysDhHuSZhZAw4SZlaozYOE75ZtVqUgXeBVZitB0mGSHsy5Ps/qY//JkpZLWpi3jzaq0z0Js4o1a0xC0mjgUuDdpPwa90iaHRF/6FV0VkScWbZe9yTMqta8XKD7AUsi4tGIeAX4ESn356A4SJhVKQK6u8ttORdozXZar9rq5fns7f2S7pV0vaQpfezfgE83zKpW/tqNRrlAy+T5/BlwbUSskXQ68H3g74sO6p6EWcUUUWoroWGez4hYERFr8tPLgH0aVeogYVa15o1J3ANMl7SzpI2A40m5P9eTtH3N06OA+xtV6tMNsyo1MYNXRKyTdCYwFxgNXBERiyVdBMyLiNnAJyUdBawDngFOblSvoo0WckhaDjxedTv6YRLwdNWNGOGG43e8U0RsU6bgxPHbxYFTP1Kq0v9++H/PbzAm0RJt1ZMo+8W2C0nzqvif1kk64jtuox/qvrRVkDDrOAF0tfetqRwkzCoVEA4SI9nMqhvQAUb+d9zmpxueAh2EiCj8ByypK19Ec5+kH0vaZKDHknSQpJ/nx0f1dfFOTdktJH18AMe4QNLny77eq8yVko7tx7GmSbqvUblG3/Gw1zO7UWariINEa70UEXtFxJ7AK8DptTuV9Pv/QUTMjoivFhTZAuh3kLCKNG+dREs4SAyd24Bd8y/o/ZL+D7AAmCLpUEl3SlqQexwTYP1lvw9Iuh34x56K8uW+l+TH20q6QdKivB0IfBXYJfdi/iOX+4Kke/Ka/Qtr6jonX1r8S2D3Rh9C0sdyPYsk/aRX7+gQSbdJekjSkbn8aEn/UXPsfx7sFzniOEiYpDHA4cDv80u7A1dFxJuA1cC5wCERsTcwD/ispPGkZbPvA94ObFen+ouB30TEDGBvYDFwFvBI7sV8QdKhwHTSVYJ7AftIeoekfUir8t5ECkJvLvFxfhoRb87Hux84tWbfNOCdwHuB7+TPcCqwMiLenOv/mKSdSxynM0RAV1e5rSIeuGytjSUtzI9vAy4HdgAej4i78uv7A3sAd0gC2Ai4E/gb4I8R8TCApGuA3lf9Qbo458MAEdEFrJS0Za8yh+btd/n5BFLQ2Ay4ISJezMeYTWN7SvoS6ZRmAml1X4/rIqIbeFjSo/kzHAq8sWa8YmI+9kMljtUZ2nzg0kGitV6KiL1qX8iBYHXtS8DNEXFCr3J70byMDAK+EhH/2esYnx7AMa4EjomIRZJOBg6q2de7rsjH/kRE1AYTJE3r53FHrjYPEj7dqN5dwFsl7QogaRNJuwEPADtL2iWXO6HO+38FnJHfO1rS5sALpF5Cj7nAP9WMdUyW9DrgVuAfJG0saTPSqU0jmwHLJI0FTuq17zhJo3KbXw88mI99Ri6PpN0kbVriOB2i5MxGhbMb7klULCKW51/kayWNyy+fGxEP5ZuK3CjpaeB2YM8+qvgUMFPSqUAXcEZE3CnpjjzFeFMel/hb4M7ck1kFfDAiFkiaBSwkXTNzW4km/xtwdy7/ezYMRg8CvwG2BU6PiJclfZc0VrFA6eDLgWPKfTsdICDafDFVW13gZdZpJo7ZJg7YvFzMnPvsd32Bl1lHavMfagcJsyr1TIG2MQcJs4pFd3uPSThImFWq2tWUZThImFWpibevaxUHCbOqtfkUqBdTmVUogOiOUlsZapwLdJykWXn/3WVWvjpImFUp8p2pymwN6NVcoIeTrgc6QdIevYqdCjwbEbsC3wC+1qheBwmzikVXV6mthDK5QI8mZe0CuB44OK+ErctjEmYVeoFn5/4yrp9Usvh4SfNqns/sdeeuvnKBvqVXHevL5DwdK4GtKUhb4CBhVqGIOKyJ1ZXJBVqmzAZ8umE2cjTMBVpbJt8MaSIpk1ddDhJmI0fDXKD5eU/KsGOBX0eDqzx9umE2QpTMBXo5cLWkJaQexPGN6vWl4mZWyKcbZlbIQcLMCjlImFkhBwkzK+QgYWaFHCTMrJCDhJkV+v82vSYzKg6iuQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Try a different method of Spam classification\n",
    "# https://towardsdatascience.com/spam-detection-with-logistic-regression-23e3709e522\n",
    "\n",
    "\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "######################\n",
    "# Even class sizes\n",
    "\n",
    "Spam_model = LogisticRegression(solver='liblinear', penalty='l1')\n",
    "Spam_model.fit(counts, y_train)\n",
    "\n",
    "example_count = vectorizer.transform(X_test)\n",
    "pred = Spam_model.predict(example_count)\n",
    "print( f'Accuracy score: {accuracy_score(y_test,pred)}' )\n",
    "f1 = f1_score(y_test, pred, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "# Accuracy of 0.66% not the best but also not bad.\n",
    "# TODO: Run this on bi/tri-grams data to see if that will be better.\n",
    "# Instead of having a matrix of the differnt words, have matrix of different ngrams.\n",
    "\n",
    "cm = confusion_matrix(y_test, pred)\n",
    "print(cm)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.9489795918367347\n",
      "F1 score: 0.4869109947643979\n",
      "[[93  0]\n",
      " [ 5  0]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAEUCAYAAADJK5isAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZ2ElEQVR4nO3de7hcVZ3m8e+bhBAg3IMIIRigA0ozBgFpwRsKjagodI+MXGTQZpoRR9qW1pZbC9rY6tPz4KVh2g5yE5CLKI+M0BOQEbkMIBBBiNyxgUgUwv0mJDnv/LH3wTqnzjm1z0lV7Tqp9/M8+6H23qvWXlWkfmettddeS7aJiGg0pe4CRETvSWCIiCYJDBHRJIEhIpokMEREkwSGiGiSwLAak3SSpPPK11tKekHS1LrLVUVj2SukPVvSyZ0uUz9JYOggSRtJulTSi5IelnRwXWWx/YjtmbZXdvpakhZIulfSgKSPd/p60X4JDJ11GvAqsClwCPCvkv603iJ1xR3Ap4BFdRckJiaBoUMkrQP8Z+AfbL9g+3rgMuDQceYzRdIJZY3jcUnfk7R+eW6uJEs6TNIjkpZJOn6UfAbTTiv3r5H0j5JukPS8pCslzWpI/zZJ/0/SM5LukLRH1TLbPs321cAfxvNZxyLpB5J+J+lZSdeOEGBnSbqq/Cw/l/SGdl27HyUwdM62wErb9zUcuwP4UwBJ7yh/dKNt7yjf8/Fyew+wNTATOHXYtd4BbAfsCXxR0psqlvFg4BPA64DpwOfKss0GLgdOBjYqj/9Q0iaVP/0oJB3c4nNvOcpb/x2YV5Z1EXD+sPOHAP8IzAJuH+F8jMO0uguwGpsJPDvs2LPAugBlDWKDCvkcApxi+yEASccCd0n6REOaL9l+GbhD0h3AfODuCnmfNRi4JF0MfLg8/jHgCttXlPtXSboV+ABwToV8R2X7+8D3J/C+MwdfSzoJeFrS+rYHv+PLbV9bnj8eeFbSHNuPrkp5+1VqDJ3zArDesGPrAc+PM5/NgYcb9h+mCOibNhz7XcPrlyiCUhWjve8NwAGNf8kpaiWbjafg7SJpqqSvSXpQ0nPAf5SnZjUkey0A2H4BeIriu4sJSGDonPuAaZLmNRybDywGkPTO8vbhaNs7y/c8RvFDHbQlsAL4fQfL/ihwru0NGrZ1bH9tVTOWdEiLzz1SU+JgYD9gL2B9YO5gdg1p5jRcYyZFE+ixVS1vv0pg6BDbLwI/Ar4saR1Jb6f4x31uef668vbhaNt1ZVYXAJ+VtFX5D/6fgItsr+hg8c8DPiTpfeVf6xmS9pC0Bbw2xuCa0d4sabqkGRQ/3DXK908pP/f5LT73IyNkuS7wCvAksDbFdzDcB8p+m+kUfQ03pxkxcQkMnfUpYC3gcYof+JG2F48zjzMpgsm1wG8oevqPamchhyt/UPsBxwFPUNQgPs8f/73MAW4YI4srgZeB3YEF5et3rUKRvkfRhPot8GvgphHSfB84kaIJsTNF30xMkDJRS4yXpNuBPW0/WXdZojMSGCKiSZoSEdEkgSEimiQwRESTBIaIaJLAEBFNEhgiokkCQ0Q0SWCIiCYJDBHRJIEhIpokMEREkwSGiGiSwBARTRIYIqJJAkNENElgiIgmCQwR0SSBISKaJDBERJMEhppJWkvS/y7XZPzBKuRziKQr21m2upRrbtxbdzn6WSaDrahcwv5o4I0Uq0ndDnylXGpuVfI9lGI6+N07vFZET5BkYJ7tB+ouS4wua1dWIOlo4Bjgk8BCiqXt96FYe2GVAgPFKlP39UNQqELStH76Lt73nnX85FMrK6W97VevLLS9T4eLVLCdbYyNYkm0F4ADxkizJvBNiiXRHitfr1me2wNYAvwdxcIzS4FPlOe+RBFklpfXOBw4CTivIe+5gIFp5f7HgYcoai2/AQ5pOH59w/t2B26hWEj3FooayeC5ayhWa7qhzOdKYNYon22w/H/fUP79KRa4vY9igZfjGtLvCtwIPFOmPRWYXp67tvwsL5af96MN+X+BYi3NcwePle/ZprzGTuX+5sAyYI+6/220Y9vpzWt6+dJtKm3Ard0qV/oYWtsNmAFcOkaa44G3ATtSrE+5K3BCw/nXUwSY2RQ//tMkbWj7RP645NxM22eMVRBJ6wDfBt5ve12KH//tI6TbiGIZ+28DGwOnAJdL2rgh2cHAJyiWlZ9OsdT9aF5P8R3MBr4InE6xIvbOwDuBL0rauky7EvgsxYKzuwF7UqzIhe3B1ajml5/3oob8N6KoPR3ReGHbD1IEjfMlrQ2cBZxt+5oxyjuJmJUeqLR1UwJDaxsDyzx29fYQ4Mu2H7f9BEVN4NCG88vL88tdLC3/ArDdBMszAOwgaS3bSz3ykncfBO63fa7tFbYvAO4BPtSQ5izb99l+GbiYIqiNZjlFf8py4EKKH/23bD9fXn8x8GYA27fZvqm87n8A/wa8u8JnOtH2K2V5hrB9OnA/cDPFitvHt8hv0jAwgCtt3ZTA0NqTwCxJY/XHjLRUfeMS7E8OCyzjWar+NS4Wyv0oRV/HUkmXS3pjhfIMlml2w/7vxlGeJ20PNoQHf7iNq22/PPh+SdtK+omk35VL1v8TQ5erH8kTtv/QIs3pwA7Av9h+pUXaScOY5V5ZaeumBIbWbqRYSHb/MdKMtFT9RJdgf5FiRedBr288aXuh7T+n+Mt5D8UPplV5Bsv02wmWaTz+laJc82yvR7EwrsZ+y9h/DstVvr8JnAGcVDaVVhupMUxCtp+laFefJml/SWtLWkPSCZKekvQAxV/jEyRtImlWmf68CV7yduBdkraUtD5w7OAJSZtK+nDZ1/AKRZNkpD8lVwDbSjpY0jRJHwW2B34ywTKNx7rAc8ALZW3myGHnfw9s3fSuZhtKelzSXcC3gNts/zeKvpPvtLPAdTKwElfauimBoQLbp1CMYTiBPy4LfyxFR9n2FFXlh4BfAXcCi4CTJ3itq4CLyrxuY+iPeQrF3Y3HKHrq303ZsTcsjyeBfcu0T1LcUdjX9rKJlGmcPkfRsfk8RW3momHnTwLOkfSMpP8yRj4vUtwSXrf87yfL40cDO0labZa578UaQwY4TYCk3YCTbL+v3D8WwPZXay3YakbSXOAntneouSgdM3/+dC+8olUXTGGzLZbeZnuXDhcJSI1homZT1BoGLWFox15EZQMVt27KyMeJGakzLVWvGDfX0H9QRQLDxCwB5jTsb8HE70JEH7Nhee/FhQSGCboFmCdpK4pbgAdSdLhFjJNY2fJubvelj2ECysFKn6Z4oOpu4OJRRiDGBEm6gGIMyXaSlkg6vO4ydYKBAVfbuik1hgkqhzZfUXc5Vle2D6q7DN3SizWGBIaIGhUDnBIYImKYAScwRESD1BgiookRyz217mI0yV2JVSDpiNapYlWs7t/xYI2hytZNCQyrZrX+R9sjVvPvWKz0lEpbN6UpEVGjYgan3vv73FOBYdZGUz13zhp1F6OyLWdPY5f5M3pwQOvo7vvV2q0T9ZAZrM162mhSfcd/4EVe9SuV6/7pfGxh7pw1+MXCOa0TxoS9b/OxpnaMdrjZV1dOa6vrzYQqeiowRPSjgdQYIqKREa+6936GvVeiiD6SzseIGNHKHhwS3XuhKqKPGLGSKZW2KiR9VtJiSXdJukDSDElbSbpZ0v2SLpI0vVU+CQwRNRvwlEpbK5JmA38D7FJOoDuVYhKhrwPfsD0PeJpimcQxJTBE1KgYEt2+GgNF98Ba5cppa1MsLPxe4JLy/DmMvXjSa5lERE3G+RDVLEm3NuwvsL3gtbzs30r6n8AjFMsGXkmxNskzDUskVprRPIEhokY24xngtGysdSUkbQjsB2wFPAP8AHj/SJdtdaEEhohaqZ0DnPYCflOuuI6kHwG7AxtImlbWGirNaJ4+hogaGdr5dOUjwNvK9VUF7An8GvgZ8JEyzWHAj1tllMAQUbN2dT7avpmik3ERxRqqU4AFwBeAo8sFmDemWDV8TGlKRNTIqK1zPto+EThx2OGHgF3Hk08CQ0TNxnErsmsSGCJq1KtzPiYwRNSoWIkqNYaIGCYzOEXEELZSY4iIZpnaLSKGKCZqSVMiIobIZLARMYwhtysjYqh2j3xslwSGiJplMtiIGKKYjyE1hogYJk2JiBii6GNIUyIihsmQ6IgYwogVA7ldGRHDZORjRAyRuxIRMaJ0PkbEEBn5GBEjSh9DRAxRTO2WwBARjZzblRExTCZqiYgRpSkREUP0ah9DR2+gStpH0r2SHpB0TCevFTFZDViVtm7qWI1B0lTgNODPgSXALZIus/3rTl0zYrLpx3EMuwIP2H4IQNKFwH4Uy3JHBIBhRZ+NfJwNPNqwvwT4sw5eL2LS6dU+hk4GhpE+rZsSSUcARwBsOTt9odF/ejEwdLIOswSY07C/BfDY8ES2F9jexfYum2zcewM9IjppsI+h1zofOxkYbgHmSdpK0nTgQOCyDl4vYlKyVWnrpo7V3W2vkPRpYCEwFTjT9uJOXS9isuq7kY+2rwCu6OQ1IiYzuzf7GNLbF1ErsXKgv25XRkQF3e4/qCKBIaJG/TiOISJacdHP0Gt6r3ET0WcGUKWtCkkbSLpE0j2S7pa0m6SNJF0l6f7yvxu2yieBIaJGpu3jGL4F/B/bbwTmA3cDxwBX254HXF3ujymBIaJW7Rv5KGk94F3AGQC2X7X9DMXDi+eUyc4B9m+VV/oYImo2MFC5NjBL0q0N+wtsL2jY3xp4AjhL0nzgNuAzwKa2lwLYXirpda0ulMAQUSN7XLcrl9neZYzz04CdgKNs3yzpW1RoNowkTYmImrXxIaolwBLbN5f7l1AEit9L2gyg/O/jrTJKYIiomV1ta52Pfwc8Kmm78tCeFBMjXQYcVh47DPhxq7zSlIioWZtHPh4FnF8+0fwQ8AmKCsDFkg4HHgEOaJVJAkNEjUx7H6m2fTswUj/EnuPJJ4EhomY9OPAxgSGiVgZXv13ZNQkMETWbVE9XlqOoRmX7ufYXJ6L/9OJDVGPVGBZTNH8aw9ngvoEtO1iuiL4w+KxErxk1MNieM9q5iGgTAz0YGCoNcJJ0oKTjytdbSNq5s8WK6B/tGuDUTi0Dg6RTgfcAh5aHXgK+08lCRfQVV9y6qMpdid1t7yTplwC2nypHVUXEKtOkvV25XNIUypglaWNgoKOliugX43u6smuq9DGcBvwQ2ETSl4Drga93tFQR/WQyNiVsf0/SbcBe5aEDbN/V2WJF9JPeqzFUHfk4FVhOEbfyqHZEO/XgAKcqdyWOBy4ANqdYsfr7ko7tdMEi+sZkbEoAHwN2tv0SgKSvUMwl99VOFiyiL0zih6geHpZuGsUEEBHRDj3YlBjrIapvUBT5JWCxpIXl/t4UdyYioh168HblWDWGwTsPi4HLG47f1LniRPQfTaYag+0zulmQiL5UQ8diFS37GCRtA3wF2B6YMXjc9rYdLFdEn1BPNiWqjEk4GziLYhTG+4GLgQs7WKaI/tKDtyurBIa1bS8EsP2g7RMonraMiHYYqLh1UZXbla9IEvCgpE8CvwVarn0XERX06EQtVQLDZ4GZwN9Q9DWsD/xVJwsV0U8m1V2JQQ3r4D3PHydriYh2mUyBQdKljFFk23/Z7sLc/+t1+cB/em+7s40hnqq7ADEJjFVjOLVrpYjoY5OqKWH76m4WJKJvTdLOx4joFNOTEyUmMETUrBebEpVnY5K0ZicLEtG3JuPIR0m7SroTuL/cny/pXzpesoh+MRkDA/BtYF/gSQDbd5Ah0RFtIVffuqlKH8MU2w8Xo6Jfs7JD5YnoP5P0rsSjknYFLGkqcBRwX2eLFdFHerDzsUpgOJKiObEl8Hvgp+WxiGgDTcbblbYfBw7sQlki+k8N/QdVVJnB6XRGqOzYPqIjJYroN5MxMFA0HQbNAP4CeLQzxYnoQ5MxMNi+qHFf0rnAVR0rUUSfaXdTorxJcCvwW9v7StqKYjrGjYBFwKG2Xx0rj4msQ7kV8IYJvC8iuuMzwN0N+18HvmF7HvA0cHirDKqMfHxa0lPl9gxFbeG4CRY4IoZr48hHSVsAHwS+W+4LeC9wSZnkHGD/VvmM2ZQoM51PMc8jwIDtHmwRRUxSHtftylmSbm3YX2B7wbA03wT+Hli33N8YeMb2inJ/CTC71YXGDAy2LelS2ztXK3dEjFv1P7XLbO8y2klJ+wKP275N0h6DhydyxSp3JX4haSfbiyqkjYhxEG3tfHw78GFJH6C4g7geRQ1iA0nTylrDFsBjrTIatY9B0mDQeAdFcLhX0iJJv5SUIBHRLm3qY7B9rO0tbM+lGJT4f20fAvwM+EiZ7DDgx63yGqvG8AtgJyp0VETEBHVn5OMXgAslnQz8Emi5Lu1YgUFQrD7VnrJFxIg6EBhsXwNcU75+CNh1PO8fKzBsIunoMS58ynguFBEjm2wPUU2lWIGq9x4Wj1id9OAAgLECw1LbX+5aSSL6UQ3TtlXRso8hIjprsj12vWfXShHRzyZTYLCdRQ4jumCy1RgiohsSGCKiUR1Tw1eRwBBRtwSGiBguNYaIaJbAEBFNEhgiYoh0PkbEiBIYImK4yfZ0ZUR0QZoSETHUJHy6MiK6IYEhIhq1eZbotklgiKhbDwaGiaxdWYmkMyU9LumuTl0jYnUgu9LWTR0LDMDZwD4dzD9i8iuXqKuydVPHAoPta4FM9hLRShsXtW2X2vsYJB0BHAEwY8rMmksT0X292PnYyaZEJbYX2N7F9i7Tp8youzgR3ZcaQ0QMkYeoImJEPRgYOnm78gLgRmA7SUskHd6pa0VMVoMDnKps3dSxGoPtgzqVd8TqRAO9V2VIUyKiTnmIKiJGkvkYIqJZagwRMVxuV0bEUAa6/IBUFQkMETVLH0NEDJGJWiKimZ2mREQ0S40hIpr1YGCo/bHriH7XrmclJM2R9DNJd0taLOkz5fGNJF0l6f7yvxu2yiuBIaJOBgZcbWttBfB3tt8EvA34H5K2B44BrrY9D7i63B9TAkNEzdo156PtpbYXla+fB+4GZgP7AeeUyc4B9m+VV/oYIupW/a7ELEm3NuwvsL1gpISS5gJvAW4GNrW9tLiUl0p6XasLJTBE1GwcdyWW2d6lZX7STOCHwN/afk7SuMuUpkREnarO91gxeEhagyIonG/7R+Xh30varDy/GfB4q3wSGCJqVIx8bM+CMyqqBmcAd9s+peHUZcBh5evDgB+3yitNiYi6te9ZibcDhwJ3Srq9PHYc8DXg4nJ6xUeAA1pllMAQUbN2LT9n+3qKSshI9hxPXgkMEXVy5TEKXZXAEFGzPCsREc3ydGVEDOFM1BIRI0mNISKa9F5cSGCIqFu7ble2UwJDRJ0MrExgiIgGotpw525LYIioWwJDRDRJYIiIIUw7H6JqmwSGiJqljyEimiUwRMQQNgz0XlsigSGibr0XFxIYIuqWPoaIaJbAEBFDDK5E1WN6KjA8t2LZsoXLFjxcdznGYRawrO5CrOYm43f8hupJnRpDK7Y3qbsM4yHp1ioLgMTE9cV3nMAQEUMYWNl7tyUSGCJqZXACw+pmxAVFo61W/++4B5sSWaJuFYy20vAgSSsl3S7pLkk/kLT2RK8laQ9JPylff1jSMWOk3UDSpyZwjZMkfa7q8WFpzpb0kXFca66ku1qla/UdT3qDdyWqbF2UwNBZL9ve0fYOwKvAJxtPqjDu/we2L7P9tTGSbACMOzBETexqWxclMHTPdcCflH8p75b0v4BFwBxJe0u6UdKismYxE0DSPpLukXQ98JeDGUn6uKRTy9ebSrpU0h3ltjvFWoXblLWVfy7TfV7SLZJ+JelLDXkdL+leST8Ftmv1IST9dZnPHZJ+OKwWtJek6yTdJ2nfMv1USf/ccO3/vqpf5GongaE/SZoGvB+4szy0HfA9228BXgROAPayvRNwK3C0pBnA6cCHgHcCrx8l+28DP7c9H9gJWAwcAzxY1lY+L2lvYB6wK7AjsLOkd0naGTgQeAtF4HlrhY/zI9tvLa93N3B4w7m5wLuBDwLfKT/D4cCztt9a5v/XkraqcJ3+YMPKldW2LkrnY2et1bDq8HUUS5RvDjxs+6by+NuA7YEbilXMmQ7cCLwR+I3t+wEknQccMcI13gv8VwDbK4FnJW04LM3e5fbLcn8mRaBYF7jU9kvlNS6r8Jl2kHQyRXNlJrCw4dzFtgeA+yU9VH6GvYE3N/Q/rF9e+74K1+oPPdj5mMDQWS/b3rHxQPnjf7HxEHCV7YOGpduR9q04IOCrtv9t2DX+dgLXOBvY3/Ydkj4O7NFwbnheLq99lO3GAIKkueO87uqrBwNDmhL1uwl4u6Q/AZC0tqRtgXuArSRtU6Y7aJT3Xw0cWb53qqT1gOcpagODFgJ/1dB3MVvS64Brgb+QtJakdSmaLa2sCyyVtAZwyLBzB0iaUpZ5a+De8tpHlumRtK2kdSpcp09UvCPR5bsSqTHUzPYT5V/eCyStWR4+wfZ9ko4ALpe0DLge2GGELD4DLJB0OLASONL2jZJuKG8H/nvZz/Am4MayxvIC8DHbiyRdBNwOPEzR3GnlH4Cby/R3MjQA3Qv8HNgU+KTtP0j6LkXfwyIVF38C2L/at9MHDO7BAU5yD1ZjIvrF+tM28W7rVYuTC5/+7m3dem4kNYaIuvXgH+cEhog6Dd6u7DEJDBE1cyaDjYihMlFLRAyXqd0iYkQ9eLsyA5wiamTAA660VVE+eHevpAfGejS/lQSGiDq5nMGpytaCpKnAaRQP7G0PHCRp+4kUK02JiJq5fbcrdwUesP0QgKQLgf2AX483owSGiBo9z9MLf+pLZlVMPkPSrQ37C4bNcDUbeLRhfwnwZxMpVwJDRI1s79PG7DTSJSaSUfoYIlYfS4A5DftbAI9NJKMEhojVxy3APElbSZpOMTtXlcl3mqQpEbGasL1C0qcp5sCYCpxpe/FE8spj1xHRJE2JiGiSwBARTRIYIqJJAkNENElgiIgmCQwR0SSBISKa/H9FwjVgL/+KFwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Try Logistical regression with unbalanced classes.\n",
    "#https://machinelearningmastery.com/what-is-imbalanced-classification/\n",
    "\n",
    "# X, y are still the full dataset.\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "vectorizer = CountVectorizer(token_pattern='(.*?,)') \n",
    "counts = vectorizer.fit_transform(X_train)\n",
    "\n",
    "Spam_model = LogisticRegression(solver='liblinear', penalty='l1')\n",
    "Spam_model.fit(counts, y_train)\n",
    "\n",
    "example_count = vectorizer.transform(X_test)\n",
    "pred = Spam_model.predict(example_count)\n",
    "print( f'Accuracy score: {accuracy_score(y_test,pred)}' )\n",
    "f1 = f1_score(y_test, pred, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "# Accuracy of 0.66% not the best but also not bad.\n",
    "# TODO: Run this on bi/tri-grams data to see if that will be better.\n",
    "# Instead of having a matrix of the differnt words, have matrix of different ngrams.\n",
    "\n",
    "cm = confusion_matrix(y_test, pred)\n",
    "print(cm)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# N-Gram data rather than target count data\n",
    "\n",
    "The algorithms used above are just using the mouse target variables to varying success.\n",
    "Try similar approaches to the balanced methods, but use bi/tri-gram data rather than just none/unigrams.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['alloc-slider-1,', 'alloc-slider-1, alloc-slider-1,', 'alloc-slider-1, alloc-slider-1, alloc-slider-1,', 'alloc-slider-1, alloc-slider-1, alloc-slider-2,', 'alloc-slider-1, alloc-slider-1, alloc-slider-3,', 'alloc-slider-1, alloc-slider-1, alloc-slider-4,', 'alloc-slider-1, alloc-slider-1, alloc-slider-5,', 'alloc-slider-1, alloc-slider-1, html,', 'alloc-slider-1, alloc-slider-2,', 'alloc-slider-1, alloc-slider-2, alloc-slider-1,', 'alloc-slider-1, alloc-slider-2, alloc-slider-2,', 'alloc-slider-1, alloc-slider-2, alloc-slider-3,', 'alloc-slider-1, alloc-slider-2, alloc-slider-4,', 'alloc-slider-1, alloc-slider-2, html,', 'alloc-slider-1, alloc-slider-3,', 'alloc-slider-1, alloc-slider-3, alloc-slider-1,', 'alloc-slider-1, alloc-slider-3, alloc-slider-2,', 'alloc-slider-1, alloc-slider-3, alloc-slider-3,', 'alloc-slider-1, alloc-slider-3, alloc-slider-4,', 'alloc-slider-1, alloc-slider-3, html,', 'alloc-slider-1, alloc-slider-4,', 'alloc-slider-1, alloc-slider-4, alloc-slider-3,', 'alloc-slider-1, alloc-slider-4, alloc-slider-4,', 'alloc-slider-1, alloc-slider-4, alloc-slider-5,', 'alloc-slider-1, alloc-slider-4, html,', 'alloc-slider-1, alloc-slider-5,', 'alloc-slider-1, alloc-slider-5, alloc-slider-1,', 'alloc-slider-1, alloc-slider-5, alloc-slider-5,', 'alloc-slider-1, alloc-slider-5, html,', 'alloc-slider-1, html,', 'alloc-slider-1, html, alloc-slider-1,', 'alloc-slider-1, html, alloc-slider-2,', 'alloc-slider-1, html, alloc-slider-3,', 'alloc-slider-1, html, alloc-slider-4,', 'alloc-slider-1, html, alloc-slider-5,', 'alloc-slider-1, html, html,', 'alloc-slider-2,', 'alloc-slider-2, alloc-slider-1,', 'alloc-slider-2, alloc-slider-1, alloc-slider-1,', 'alloc-slider-2, alloc-slider-1, alloc-slider-2,', 'alloc-slider-2, alloc-slider-1, alloc-slider-4,', 'alloc-slider-2, alloc-slider-1, html,', 'alloc-slider-2, alloc-slider-2,', 'alloc-slider-2, alloc-slider-2, alloc-slider-1,', 'alloc-slider-2, alloc-slider-2, alloc-slider-2,', 'alloc-slider-2, alloc-slider-2, alloc-slider-3,', 'alloc-slider-2, alloc-slider-2, alloc-slider-4,', 'alloc-slider-2, alloc-slider-2, alloc-slider-5,', 'alloc-slider-2, alloc-slider-2, html,', 'alloc-slider-2, alloc-slider-3,', 'alloc-slider-2, alloc-slider-3, alloc-slider-1,', 'alloc-slider-2, alloc-slider-3, alloc-slider-2,', 'alloc-slider-2, alloc-slider-3, alloc-slider-3,', 'alloc-slider-2, alloc-slider-3, alloc-slider-4,', 'alloc-slider-2, alloc-slider-3, alloc-slider-5,', 'alloc-slider-2, alloc-slider-3, html,', 'alloc-slider-2, alloc-slider-4,', 'alloc-slider-2, alloc-slider-4, alloc-slider-1,', 'alloc-slider-2, alloc-slider-4, alloc-slider-3,', 'alloc-slider-2, alloc-slider-4, alloc-slider-4,', 'alloc-slider-2, alloc-slider-4, alloc-slider-5,', 'alloc-slider-2, alloc-slider-4, html,', 'alloc-slider-2, alloc-slider-5,', 'alloc-slider-2, alloc-slider-5, alloc-slider-4,', 'alloc-slider-2, alloc-slider-5, alloc-slider-5,', 'alloc-slider-2, html,', 'alloc-slider-2, html, alloc-slider-1,', 'alloc-slider-2, html, alloc-slider-2,', 'alloc-slider-2, html, alloc-slider-3,', 'alloc-slider-2, html, alloc-slider-4,', 'alloc-slider-2, html, alloc-slider-5,', 'alloc-slider-2, html, html,', 'alloc-slider-3,', 'alloc-slider-3, alloc-slider-1,', 'alloc-slider-3, alloc-slider-1, alloc-slider-1,', 'alloc-slider-3, alloc-slider-1, alloc-slider-2,', 'alloc-slider-3, alloc-slider-1, html,', 'alloc-slider-3, alloc-slider-2,', 'alloc-slider-3, alloc-slider-2, alloc-slider-1,', 'alloc-slider-3, alloc-slider-2, alloc-slider-2,', 'alloc-slider-3, alloc-slider-2, alloc-slider-3,', 'alloc-slider-3, alloc-slider-2, alloc-slider-4,', 'alloc-slider-3, alloc-slider-2, html,', 'alloc-slider-3, alloc-slider-3,', 'alloc-slider-3, alloc-slider-3, alloc-slider-1,', 'alloc-slider-3, alloc-slider-3, alloc-slider-2,', 'alloc-slider-3, alloc-slider-3, alloc-slider-3,', 'alloc-slider-3, alloc-slider-3, alloc-slider-4,', 'alloc-slider-3, alloc-slider-3, alloc-slider-5,', 'alloc-slider-3, alloc-slider-3, html,', 'alloc-slider-3, alloc-slider-4,', 'alloc-slider-3, alloc-slider-4, alloc-slider-1,', 'alloc-slider-3, alloc-slider-4, alloc-slider-2,', 'alloc-slider-3, alloc-slider-4, alloc-slider-3,', 'alloc-slider-3, alloc-slider-4, alloc-slider-4,', 'alloc-slider-3, alloc-slider-4, alloc-slider-5,', 'alloc-slider-3, alloc-slider-4, html,', 'alloc-slider-3, alloc-slider-5,', 'alloc-slider-3, alloc-slider-5, alloc-slider-4,', 'alloc-slider-3, alloc-slider-5, alloc-slider-5,', 'alloc-slider-3, alloc-slider-5, html,', 'alloc-slider-3, html,', 'alloc-slider-3, html, alloc-slider-1,', 'alloc-slider-3, html, alloc-slider-2,', 'alloc-slider-3, html, alloc-slider-3,', 'alloc-slider-3, html, alloc-slider-4,', 'alloc-slider-3, html, alloc-slider-5,', 'alloc-slider-3, html, html,', 'alloc-slider-4,', 'alloc-slider-4, alloc-slider-1,', 'alloc-slider-4, alloc-slider-1, alloc-slider-1,', 'alloc-slider-4, alloc-slider-1, html,', 'alloc-slider-4, alloc-slider-2,', 'alloc-slider-4, alloc-slider-2, alloc-slider-1,', 'alloc-slider-4, alloc-slider-2, alloc-slider-2,', 'alloc-slider-4, alloc-slider-2, alloc-slider-3,', 'alloc-slider-4, alloc-slider-2, alloc-slider-4,', 'alloc-slider-4, alloc-slider-2, html,', 'alloc-slider-4, alloc-slider-3,', 'alloc-slider-4, alloc-slider-3, alloc-slider-1,', 'alloc-slider-4, alloc-slider-3, alloc-slider-2,', 'alloc-slider-4, alloc-slider-3, alloc-slider-3,', 'alloc-slider-4, alloc-slider-3, alloc-slider-4,', 'alloc-slider-4, alloc-slider-3, html,', 'alloc-slider-4, alloc-slider-4,', 'alloc-slider-4, alloc-slider-4, alloc-slider-1,', 'alloc-slider-4, alloc-slider-4, alloc-slider-2,', 'alloc-slider-4, alloc-slider-4, alloc-slider-3,', 'alloc-slider-4, alloc-slider-4, alloc-slider-4,', 'alloc-slider-4, alloc-slider-4, alloc-slider-5,', 'alloc-slider-4, alloc-slider-4, html,', 'alloc-slider-4, alloc-slider-5,', 'alloc-slider-4, alloc-slider-5, alloc-slider-1,', 'alloc-slider-4, alloc-slider-5, alloc-slider-3,', 'alloc-slider-4, alloc-slider-5, alloc-slider-4,', 'alloc-slider-4, alloc-slider-5, alloc-slider-5,', 'alloc-slider-4, alloc-slider-5, html,', 'alloc-slider-4, html,', 'alloc-slider-4, html, alloc-slider-1,', 'alloc-slider-4, html, alloc-slider-2,', 'alloc-slider-4, html, alloc-slider-3,', 'alloc-slider-4, html, alloc-slider-4,', 'alloc-slider-4, html, alloc-slider-5,', 'alloc-slider-4, html, html,', 'alloc-slider-5,', 'alloc-slider-5, alloc-slider-1,', 'alloc-slider-5, alloc-slider-1, alloc-slider-1,', 'alloc-slider-5, alloc-slider-1, alloc-slider-2,', 'alloc-slider-5, alloc-slider-1, alloc-slider-3,', 'alloc-slider-5, alloc-slider-1, html,', 'alloc-slider-5, alloc-slider-2,', 'alloc-slider-5, alloc-slider-2, alloc-slider-1,', 'alloc-slider-5, alloc-slider-2, alloc-slider-2,', 'alloc-slider-5, alloc-slider-2, alloc-slider-5,', 'alloc-slider-5, alloc-slider-2, html,', 'alloc-slider-5, alloc-slider-3,', 'alloc-slider-5, alloc-slider-3, alloc-slider-2,', 'alloc-slider-5, alloc-slider-3, alloc-slider-3,', 'alloc-slider-5, alloc-slider-3, alloc-slider-4,', 'alloc-slider-5, alloc-slider-3, html,', 'alloc-slider-5, alloc-slider-4,', 'alloc-slider-5, alloc-slider-4, alloc-slider-2,', 'alloc-slider-5, alloc-slider-4, alloc-slider-3,', 'alloc-slider-5, alloc-slider-4, alloc-slider-4,', 'alloc-slider-5, alloc-slider-4, alloc-slider-5,', 'alloc-slider-5, alloc-slider-4, html,', 'alloc-slider-5, alloc-slider-5,', 'alloc-slider-5, alloc-slider-5, alloc-slider-1,', 'alloc-slider-5, alloc-slider-5, alloc-slider-2,', 'alloc-slider-5, alloc-slider-5, alloc-slider-3,', 'alloc-slider-5, alloc-slider-5, alloc-slider-4,', 'alloc-slider-5, alloc-slider-5, alloc-slider-5,', 'alloc-slider-5, alloc-slider-5, html,', 'alloc-slider-5, html,', 'alloc-slider-5, html, alloc-slider-1,', 'alloc-slider-5, html, alloc-slider-2,', 'alloc-slider-5, html, alloc-slider-3,', 'alloc-slider-5, html, alloc-slider-4,', 'alloc-slider-5, html, alloc-slider-5,', 'alloc-slider-5, html, html,', 'html,', 'html, alloc-slider-1,', 'html, alloc-slider-1, alloc-slider-1,', 'html, alloc-slider-1, alloc-slider-2,', 'html, alloc-slider-1, alloc-slider-3,', 'html, alloc-slider-1, alloc-slider-5,', 'html, alloc-slider-1, html,', 'html, alloc-slider-2,', 'html, alloc-slider-2, alloc-slider-1,', 'html, alloc-slider-2, alloc-slider-2,', 'html, alloc-slider-2, alloc-slider-3,', 'html, alloc-slider-2, alloc-slider-4,', 'html, alloc-slider-2, alloc-slider-5,', 'html, alloc-slider-2, html,', 'html, alloc-slider-3,', 'html, alloc-slider-3, alloc-slider-1,', 'html, alloc-slider-3, alloc-slider-2,', 'html, alloc-slider-3, alloc-slider-3,', 'html, alloc-slider-3, alloc-slider-4,', 'html, alloc-slider-3, alloc-slider-5,', 'html, alloc-slider-3, html,', 'html, alloc-slider-4,', 'html, alloc-slider-4, alloc-slider-1,', 'html, alloc-slider-4, alloc-slider-2,', 'html, alloc-slider-4, alloc-slider-3,', 'html, alloc-slider-4, alloc-slider-4,', 'html, alloc-slider-4, alloc-slider-5,', 'html, alloc-slider-4, html,', 'html, alloc-slider-5,', 'html, alloc-slider-5, alloc-slider-1,', 'html, alloc-slider-5, alloc-slider-2,', 'html, alloc-slider-5, alloc-slider-3,', 'html, alloc-slider-5, alloc-slider-4,', 'html, alloc-slider-5, alloc-slider-5,', 'html, alloc-slider-5, html,', 'html, html,', 'html, html, alloc-slider-1,', 'html, html, alloc-slider-2,', 'html, html, alloc-slider-3,', 'html, html, alloc-slider-4,', 'html, html, alloc-slider-5,', 'html, html, html,']\n"
     ]
    }
   ],
   "source": [
    "# TODO get a list of bi grams for each user.\n",
    "# Turns out the Count Vectorizer has an ngram feature :/\n",
    "\n",
    "# First do with unbalanced classes, then balanced.\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y)\n",
    "\n",
    "vectorizer_ngram = CountVectorizer(token_pattern='(.*?,)', ngram_range=(1, 3))\n",
    "ngram_counts = vectorizer_ngram.fit_transform(X_train)\n",
    "print(vectorizer_ngram.get_feature_names()) # Not sure if this worked correclty, lets try anyway lol.\n",
    "\n",
    "X_test = vectorizer_ngram.transform(X_test) # TODO: SHould be treating alloc-slider-1/alloc-slider-2 as different n-grams.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "222"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vectorizer_ngram.get_feature_names())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy score: 0.3877551020408163\n",
      "F1 score: 0.3244485294117647\n",
      "[[34 59]\n",
      " [ 1  4]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQYAAAEOCAYAAABrZrlRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAZuUlEQVR4nO3dfbRcVZnn8e8vCSRAAgECARMwEQNKMwYhMgi+IMiLiEL3SItEJtC0GXB0bGntFqE12tg6q9dS24FRgygBBIKtWTCAHTAj8jKAJJgoMZAAgkQiIeFFAojJvc/8cfaVulX3Vp26t06durd+n7XOos5L7fNUkXruPnufs7ciAjOzSmPKDsDMOo8Tg5nVcGIwsxpODGZWw4nBzGo4MZhZDSeGUUzSAklXpdf7StoiaWzZceVRGXuOYy+XdFHRMXUTJ4YCSdpN0hJJL0p6XNLpZcUSEb+NiIkR0VP0uSQtlPSQpF5JZxZ9Pms9J4ZiXQL8CZgKzAW+Kekvyg2pLVYBHwXuLzsQGxonhoJI2gn4L8A/RcSWiLgTuAE4o8lyxki6MNU4Nkq6QtIuad8MSSFpnqTfStok6YJByuk7dlxav03SP0u6S9ILkm6RNKXi+MMl/T9Jz0laJemovDFHxCURsQz4YzOftR5JP5D0e0nPS7p9gAQ7RdKt6bP8TNJrW3XubuTEUJz9gZ6IWFuxbRXwFwCS3pZ+dIMtb0vvOTMt7wJeB0wELq4619uAA4BjgM9JemPOGE8HzgL2BLYHPpVimwbcBFwE7Ja2/1DSHrk//SAknd7gc+87yFt/DMxKsd4PfL9q/1zgn4EpwMoB9lsTxpUdwCg2EXi+atvzwCSAVIOYnKOcucBXI+JRAEnnAw9IOqvimC9ExMvAKkmrgNnAmhxlf68vcUm6Dnh/2v5h4OaIuDmt3yppOXAisChHuYOKiKuBq4fwvu/2vZa0AHhW0i4R0fcd3xQRt6f9FwDPS9onIp4YTrzdyjWG4mwBdq7atjPwQpPlvAZ4vGL9cbKEPrVi2+8rXr9ElpTyGOx9rwVOrfxLTlYr2buZwFtF0lhJX5H0iKQ/AI+lXVMqDvtzAoiILcAzZN+dDYETQ3HWAuMkzarYNhtYDSDp7an7cLDl7ek9T5L9UPvsC2wDniow9ieAKyNicsWyU0R8ZbgFS5rb4HMPdClxOnAy8G5gF2BGX3EVx+xTcY6JZJdATw433m7lxFCQiHgR+BHwRUk7STqS7B/3lWn/Han7cLDljlTUNcAnJc1M/+D/BVgcEdsKDP8q4H2Sjk9/rSdIOkrSdPjzPQa3DfZmSdtLmkD2w90uvX9M+tzfb/C5fztAkZOAV4DNwI5k30G1E1O7zfZkbQ33+jJi6JwYivVRYAdgI9kP/NyIWN1kGd8lSya3A78ha+n/eCuDrJZ+UCcDnwWeJqtBfJpX/73sA9xVp4hbgJeBI4CF6fU7hhHSFWSXUL8Dfg3cM8AxVwOfJ7uEOJSsbcaGSB6oxZolaSVwTERsLjsWK4YTg5nV8KWEmdVwYjCzGk4MZlbDicHMajgxmFkNJwYzq+HEYGY1nBjMrIYTg5nVcGIwsxpODGZWw4nBzGo4MZhZDScGM6vhxGBmNZwYzKyGE4OZ1XBiMLMaTgxmVsOJoWSSdpD0f9KcjD8YRjlzJd3SytjKkubceKjsOLqZB4PNKU1hfx7wBrLZpFYCX0pTzQ2n3DPIhoM/ouC5IjqCpABmRcTDZcdig/PclTlIOg/4DHAOsJRsavsTyOZeGFZiIJtlam03JIU8JI3rpu/i+HftFJuf6cl17IpfvrI0Ik4oOKRMRHips5BNibYFOLXOMeOBr5NNifZkej0+7TsKWA/8PdnEMxuAs9K+L5Alma3pHGcDC4CrKsqeAQQwLq2fCTxKVmv5DTC3YvudFe87AriPbCLd+8hqJH37biObremuVM4twJRBPltf/P9QEf8pZBPcriWb4OWzFccfBtwNPJeOvRjYPu27PX2WF9Pn/WBF+f9INpfmlX3b0nv2S+c4JK2/BtgEHFX2v41WLIe8aXxs3bBfrgVY3q643MbQ2FuBCcCSOsdcABwOHEw2P+VhwIUV+/ciSzDTyH78l0jaNSI+z6tTzk2MiMvqBSJpJ+AbwHsiYhLZj3/lAMftRjaN/TeA3YGvAjdJ2r3isNOBs8imld+ebKr7wexF9h1MAz4HXEo2I/ahwNuBz0l6XTq2B/gk2YSzbwWOIZuRi4jom41qdvq8iyvK342s9jS/8sQR8QhZ0vi+pB2B7wGXR8RtdeIdQYKe6M21tJMTQ2O7A5uifvV2LvDFiNgYEU+T1QTOqNi/Ne3fGtnU8luAA4YYTy9wkKQdImJDDDzl3XuBdRFxZURsi4hrgAeB91Uc872IWBsRLwPXkSW1wWwla0/ZClxL9qP/t4h4IZ1/NfAmgIhYERH3pPM+BnwbeGeOz/T5iHglxdNPRFwKrAPuJZtx+4IG5Y0YAfQSuZZ2cmJobDMwRVK99piBpqqvnIJ9c1ViaWaq+j+LbKLcD5K1dWyQdJOkN+SIpy+maRXrv28ins0R0Xch3PfDrZxt++W+90vaX9KNkn6fpqz/F/pPVz+QpyPijw2OuRQ4CPhfEfFKg2NHjCDYGj25lnZyYmjsbrKJZE+pc8xAU9UPdQr2F8lmdO6zV+XOiFgaEceS/eV8kOwH0yievph+N8SYmvFNsrhmRcTOZBPjqv5b6v85TLN8fx24DFiQLpVGDdcYRqCIeJ7suvoSSadI2lHSdpIulPSMpIfJ/hpfKGkPSVPS8VcN8ZQrgXdI2lfSLsD5fTskTZX0/tTW8ArZJclAf0puBvaXdLqkcZI+CBwI3DjEmJoxCfgDsCXVZs6t2v8U8Lqad9XaVdJGSQ8A/wasiIi/JWs7+VYrAy5TAD1ErqWdnBhyiIivkt3DcCGvTgt/PllD2YFkVeVHgV8CvwLuBy4a4rluBRanslbQ/8c8hqx340mylvp3khr2qsrYDJyUjt1M1qNwUkRsGkpMTfoUWcPmC2S1mcVV+xcAiyQ9J+mv65TzIlmX8KT033PS9vOAQySNmmnuO7HG4BuchkDSW4EFEXF8Wj8fICK+XGpgo4ykGcCNEXFQyaEUZvbs7WPpzY2aYDJ7T9+wIiLmFBwS4BrDUE0jqzX0WU//hj2z3HpzLu3kOx+HZqDGNFe9rGlRQvtBHk4MQ7Me2KdifTpD74WwLhYBWzsvLzgxDNF9wCxJM8m6AE8ja3Aza5Loadib235uYxiCdLPSx8geqFoDXDfIHYg2RJKuIbuH5ABJ6yWdXXZMRQigN/It7eQawxClW5tvLjuO0SoiPlR2DO3SiTUGJwazEmU3ODkxmFmV3nBiMLMKrjGYWY1AbI2xZYdRw70SwyBpfuOjbDhG+3fcV2PIs7STE8PwjOp/tB1ilH/HoifG5FrayZcSZiXKRnDqvL/PHZUYxk7aKcZN2bXsMHIbu/tkxs+c3oE3tA7uP+3cjievW2ffaeOYM3vCiPqOH3tiK5ue6cld93fjYwPjpuzKXgs+VnYYo9rPj6873qy1wGHHP9H4oCRCbb9MyKOjEoNZN+p1jcHMKgXiT9F5P8POi8isi7jx0cwG1ONbos2sUiB6XGMws2q97pUws0rZLdFODGZWodUPUUl6jGxOjx5gW0TMSTN3LSabOf0x4K8j4tl65XReqjLrIhEU8azEuyLi4Io5KD4DLIuIWcCytF6XE4NZqURvzmUYTgYWpdeLqD8PK+BLCbNSBTRTG5giaXnF+sKIWDhAkbdICuDbaf/UiNgAEBEbJO3Z6ERODGYla6LxcVOOKeqOjIgn04//VkkPDiUmJwazEgVq6ZiPEfFk+u9GSUuAw4CnJO2dagt7AxsbleM2BrOS9TAm19KIpJ0kTep7DRwHPADcAMxLh80Drm9UlmsMZiVqcXflVGCJJMh+21dHxH9Iug+4Lk3a81vg1EYFOTGYlSibiao1FfeIeBSYPcD2zcAxzZTlxGBWMo/gZGb9RMjPSphZLQ/tZmb9ZAO1+FLCzPrxYLBmViWgI6eoc2IwK1Gr73xsFScGs5J5MFgz6ycbj8E1BjOr4ksJM+sna2PwpYSZVfEt0WbWTyC29bq70syq+M5HM+vHvRJmNiA3PppZP77z0cwG5DYGM+snG9rNicHMKoW7K82sigdqMbMB+VLCzPrp1DaGQjtQJZ0g6SFJD0tqOPW2WTfqDeVa2qmwGoOkscAlwLHAeuA+STdExK+LOqfZSNON9zEcBjycZsdB0rXAyYATg1mfgG1ddufjNOCJivX1wH8u8HxmI06ntjEUmRgG+rRRc5A0H5gPMHb3yQWGY9aZui0xrAf2qVifDjxZfVBELAQWAoyfOb0mcZiNZp3axlDkxc19wCxJMyVtD5wG3FDg+cxGpAjlWvKSNFbSLyTdmNZnSrpX0jpJi9Pvsa7CEkNEbAM+BiwF1gDXRcTqos5nNlL1olxLEz5B9pvr8z+Br0XELOBZ4OxGBRTaHBoRN0fE/hGxX0R8qchzmY1EEa29j0HSdOC9wHfSuoCjgX9PhywCTmlUju98NCuV6Olt6d/nrwP/AExK67sDz6UaPGRtf9MaFdJ5HahmXaaJNoYpkpZXLPMry5F0ErAxIlZUbh7olI1ico3BrERN3sewKSLm1Nl/JPB+SScCE4CdyWoQkyWNS7WGAXsHq7nGYFamyNoZ8iwNi4o4PyKmR8QMsl7A/xsRc4GfAh9Ih80Drm9UlhODWckK6JWo9o/AeZIeJmtzuKzRG3wpYVaigKbuUchdbsRtwG3p9aNkzy7l5sRgVqrOvPPRicGsZL29TgxmViFrWHRiMLMqvpQwsxp5uiLbzYnBrGS+lDCzfoLmHqluFycGs5J14JWEE4NZqQLC3ZVmVm1EXUpI2rneGyPiD60Px6z7jLReidVklz+V6axvPYB9C4zLrCsU9azEcA2aGCJin8H2mVmLBNCBiSHXY9eSTpP02fR6uqRDiw3LrHu0ajyGVmqYGCRdDLwLOCNtegn4VpFBmXWVyLm0UZ5eiSMi4hBJvwCIiGfyjEtvZnloxHZXbpU0hpSzJO0O9BYalVm36NCnK/O0MVwC/BDYQ9IXgDvJJrAws1YYiZcSEXGFpBXAu9OmUyPigWLDMusmnVdjyHvn41hgK1ne8gCyZq3UgTc45emVuAC4BngN2Zj0V0s6v+jAzLrGSLyUAD4MHBoRLwFI+hKwAvhykYGZdYUR/BDV41XHjQMeLSYcsy7UgZcS9R6i+hpZyC8BqyUtTevHkfVMmFkrdGB3Zb0aQ1/Pw2rgport9xQXjln30UiqMUREw2mszGyYSmhYzKNhG4Ok/YAvAQeSzaALQETsX2BcZl1CHXkpkeeehMuB75HdhfEe4Drg2gJjMusuHdhdmScx7BgRSwEi4pGIuJDsaUsza4XenEsb5emufEWSgEcknQP8Dtiz2LDMusQIHqjlk8BE4H8ARwIfAf6myKDMuoki39KwHGmCpJ9LWiVpdXroEUkzJd0raZ2kxXmGTcjzENW96eULvDpYi5m1SuvaD14Bjo6ILZK2A+6U9GPgPOBrEXGtpG8BZwPfrFdQvRucltQLOSL+akih1zH+sZfY/6wVrS7WKpy4+9FlhzDqrXvuR6WcNyIC2JJWt0tLAEcDp6fti4AFDDUxABcPK0ozy6WJG5ymSFpesb4wIhb2K0saS/Ys0+vJxlJ5BHguIralQ9YD0xqdqN4NTstyh2tmQ5e/8XFTRMypW1RED3CwpMnAEuCNAx3W6ESeicqsTEEhXZER8Zyk24DDgcmSxqVaw3TgyUbv96ArZiVrYa/EHqmmgKQdyEZdWwP8FPhAOmwecH2jsnLXGCSNj4hX8h5vZjm1rldib2BRamcYA1wXETdK+jVwraSLgF8ADZ+DyvOsxGGpoF2AfSXNBv42Ij4+nE9gZkmLEkNE/BJ48wDbHwUOa6asPJcS3wBOAjank6zCt0SbtUTey4h2P5qd51JiTEQ8nt0V/Wc9BcVj1n068JboPInhiXQ5Eena5ePA2mLDMusiI3E8BuBcssuJfYGngJ+kbWbWAurAed3yPCuxETitDbGYdZ8S2g/yyNMrcSkDVHYiYn4hEZl1m5GYGMguHfpMAP4SeKKYcMy60EhMDBGxuHJd0pXArYVFZNZlOvFSYii3RM8EXtvqQMysc+RpY3iWVys7Y4BngM8UGZRZV+nAGkPdxJDGepxNNs4jQG8aDMLMWiE6s7uy7qVESgJLIqInLU4KZq02QoeP/7mkQwqPxKwLiRH2rETFwA5vAz4i6RHgRbLPEhHhZGHWCh1YD6/XxvBz4BDglDbFYtZ9RuCdj4Js9qk2xWLWnUZYYthD0nmD7YyIrxYQj1nX6cReiXqJYSzZDFSd97C42WgywmoMGyLii22LxKwbldAVmUfDNgYzK9ZIa3w8pm1RmHWzkZQYIuKZdgZi1q1GWo3BzNrBicHMKpVxu3MeTgxmZXNiMLNqrjGYWS0nBjOr4cRgZv10aOPjUAaDNbNWatEITpL2kfRTSWskrZb0ibR9N0m3SlqX/rtro7KcGMxKpt58Sw7bgL+PiDcChwP/XdKBZIM3L4uIWcAycgzm7MRgVrJWDe0WERsi4v70+gVgDTANOBlYlA5bRI7Bl9zGYFamgp6ulDQDeDNwLzA1IjZAljwk7dno/U4MZmXLnximSFpesb4wIhZWHyRpIvBD4O8i4g/ZLBDNcWIwK1HfKNE5bYqIOXXLk7YjSwrfj4gfpc1PSdo71Rb2BjY2OpHbGMzK1rpeCQGXAWuqhl68AZiXXs8Drm9UVmGJQdJ3JW2U9EBR5zAbDRSRa8nhSOAM4GhJK9NyIvAV4FhJ64Bj03pdRV5KXA5cDFxR4DnMRrYWTlEXEXcy+MhrTQ28VFiNISJuJ5sA18zq6cAp6kpvfJQ0H5gPMIEdS47GrP18S/QAImJhRMyJiDnbMb7scMzazzUGM+unQx+icmIwK1sHJoYiuyuvAe4GDpC0XtLZRZ3LbKTqu8GpFc9KtFJhNYaI+FBRZZuNJurtvCqDLyXMyjQCp6gzszYYabNdm1k7uMZgZtXcXWlm/QWQ7wGptnJiMCuZ2xjMrJ8mB2ppGycGszJF+FLCzGq5xmBmtZwYzKyaawxm1l8AflbCzKq5u9LMarlXwsyquY3BzPrzY9dmVi2787HzMoMTg1nZ3PhoZtVcYzCz/iJ8H4OZ1XKvhJnV8qWEmfXTwtmuW6n0uSvNul7fmAyNlhwkfVfSRkkPVGzbTdKtktal/+7aqBwnBrOytXZS28uBE6q2fQZYFhGzgGVpvS4nBrOSKSLXkkdE3A48U7X5ZGBRer0IOKVROW5jMCtTAD25qwNTJC2vWF8YEQtzvG9qRGwAiIgNkvZs9AYnBrMSify1AWBTRMwpMp4+vpQwK1sLGx8H8ZSkvQHSfzc2eoMTg1nZik8MNwDz0ut5wPWN3uDEYFamIHuIKs+Sg6RrgLuBAyStl3Q28BXgWEnrgGPTel1uYzArWSsfooqIDw2y65hmynFiMCubb4k2s34ioLfz7ol2YjArW+flBScGs7J5oBYzq+XEYGb9eCaqxl7g2U0/iX9/vOw4mjAF2FR2EE0ZWdHCSPyO4bX5Dx32zUuF6KjEEBF7lB1DMyQtb9e9692qK75jJwYz6yeAns7rlnBiMCtVQDgxjDZ5noW34Rn933EHXkr4IaphaDRIhqQeSSslPSDpB5J2HOq5JB0l6cb0+v2SBh2eS9JkSR8dwjkWSPpU3u1Vx1wu6QNNnGtG5biEg8k5EMnI1dcrkWdpIyeGYr0cEQdHxEHAn4BzKncq0/T/g4i4ISLqPSE3GWg6MVhJin/sumlODO1zB/D69JdyjaT/DdwP7CPpOEl3S7o/1SwmAkg6QdKDku4E/qqvIElnSro4vZ4qaYmkVWk5guyx2v1SbeVf03GflnSfpF9K+kJFWRdIekjST4ADGn0ISR9J5ayS9MOqWtC7Jd0haa2kk9LxYyX9a8W5/9twv8hRx4mhO0kaB7wH+FXadABwRUS8GXgRuBB4d0QcAiwHzpM0AbgUeB/wdmCvQYr/BvCziJgNHAKsJhsF+JFUW/m0pOOAWcBhwMHAoZLeIelQ4DTgzWSJ5y05Ps6PIuIt6XxrgLMr9s0A3gm8F/hW+gxnA89HxFtS+R+RNDPHebpDBPT05FvayI2PxdpB0sr0+g7gMuA1wOMRcU/afjhwIHCXJIDtyQbaeAPwm4hYByDpKmD+AOc4GvivABHRAzw/wLwBx6XlF2l9IlmimAQsiYiX0jluyPGZDpJ0EdnlykRgacW+6yKiF1gn6dH0GY4D3lTR/rBLOvfaHOfqDh3Y+OjEUKyXI+Lgyg3px/9i5Sbg1uoBNiQdTDOzCdQn4MsR8e2qc/zdEM5xOXBKRKySdCZwVMW+6rIinfvjEVGZQJA0o8nzjl4dmBh8KVG+e4AjJb0eQNKOkvYHHgRmStovHTfYyDzLgHPTe8dK2hl4gaw20Gcp8DcVbRfT0hDitwN/KWkHSZPILlsamQRskLQdMLdq36mSxqSYXwc8lM59bjoeSftL2inHebpEzh6JNvdKuMZQsoh4Ov3lvUbS+LT5wohYK2k+cJOkTcCdwEEDFPEJYGEa268HODci7pZ0V+oO/HFqZ3gjcHeqsWwBPhwR90taDKwEHie73Gnkn4B70/G/on8Cegj4GTAVOCci/ijpO2RtD/crO/nT5JjwpGsERAfe4KTowGqMWbfYZdwe8dad8+XJpc9+Z0W7nhtxjcGsbB34x9mJwaxMfd2VHcaJwaxk4cFgzaw/D9RiZtU8tJuZDagDuyudGMxKFEC4xmBm/YRHcDKzAUQHdlf6zkezEkn6D7Ih8vPYFBEnFBlPHycGM6vhpyvNrIYTg5nVcGIwsxpODGZWw4nBzGr8f3i/PYkJyjD1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 288x288 with 2 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "ngram_model = MultinomialNB()\n",
    "ngram_model.fit(ngram_counts, y_train)\n",
    "\n",
    "pred = ngram_model.predict(X_test)\n",
    "print( f'Accuracy score: {accuracy_score(y_test, pred)}' )\n",
    "f1 = f1_score(y_test, pred, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "# Accuracy of 0.96 on unbalanced data\n",
    "\n",
    "cm = confusion_matrix(y_test, pred)\n",
    "print(cm)\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Try with n-grams and balanced data.\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_balanced, y_balanced)\n",
    "\n",
    "vectorizer_ngram = CountVectorizer(token_pattern='(.*?,)', ngram_range=(2, 2))\n",
    "ngram_counts = vectorizer_ngram.fit_transform(X_train)\n",
    "print(vectorizer_ngram.get_feature_names()) # Not sure if this worked correclty, lets try anyway lol.\n",
    "\n",
    "X_test = vectorizer_ngram.transform(X_test)\n",
    "\n",
    "Spam_model = LogisticRegression(solver='liblinear', penalty='l1')\n",
    "Spam_model.fit(ngram_counts, y_train)\n",
    "\n",
    "pred = Spam_model.predict(X_test)\n",
    "print( f'\\nAccuracy score: {accuracy_score(y_test, pred)}' )\n",
    "f1 = f1_score(y_test, pred, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "# Accuracy of 0.777 on balanced data\n",
    "# Prety good, I imagine if this was working proberly itd be even better\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_train, Spam_model.predict(ngram_counts))\n",
    "print('Train data matrix: \\n', cm)\n",
    "cm = confusion_matrix(y_test, pred)\n",
    "print('Test data matrix: \\n', cm)\n",
    "\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BALENCED TRIGRAMS\n",
    "\n",
    "# Try with n-grams and balanced data.\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_balanced, y_balanced)\n",
    "\n",
    "vectorizer_ngram = CountVectorizer(token_pattern='(.*?,)', ngram_range=(3, 3))\n",
    "ngram_counts = vectorizer_ngram.fit_transform(X_train)\n",
    "print(vectorizer_ngram.get_feature_names()[:10]) # Not sure if this worked correclty, lets try anyway lol.\n",
    "\n",
    "X_test = vectorizer_ngram.transform(X_test)\n",
    "\n",
    "Spam_model = LogisticRegression(solver='liblinear', penalty='l1')\n",
    "Spam_model.fit(ngram_counts, y_train)\n",
    "\n",
    "pred = Spam_model.predict(X_test)\n",
    "print( f'\\nAccuracy score: {accuracy_score(y_test, pred)}' )\n",
    "f1 = f1_score(y_test, pred, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "# Accuracy of 0.777 on balanced data\n",
    "# Prety good, I imagine if this was working proberly itd be even better\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_train, Spam_model.predict(ngram_counts))\n",
    "print('Train data matrix: \\n', cm)\n",
    "cm = confusion_matrix(y_test, pred)\n",
    "print('Test data matrix: \\n', cm)\n",
    "\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# BALENCED UNI and BiGRAMS\n",
    "\n",
    "# Try with n-grams and balanced data.\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(X_balanced, y_balanced)\n",
    "\n",
    "vectorizer_ngram = CountVectorizer(token_pattern='(.*?,)', ngram_range=(1, 2))\n",
    "ngram_counts = vectorizer_ngram.fit_transform(X_train)\n",
    "print(vectorizer_ngram.get_feature_names()[:10]) # Not sure if this worked correclty, lets try anyway lol.\n",
    "\n",
    "X_test = vectorizer_ngram.transform(X_test)\n",
    "\n",
    "Spam_model = LogisticRegression(solver='liblinear', penalty='l1')\n",
    "Spam_model.fit(ngram_counts, y_train)\n",
    "\n",
    "pred = Spam_model.predict(X_test)\n",
    "print( f'\\nAccuracy score: {accuracy_score(y_test, pred)}' )\n",
    "f1 = f1_score(y_test, pred, average='macro')\n",
    "print( f'F1 score: {f1}' )\n",
    "# Accuracy of 0.777 on balanced data\n",
    "# Prety good, I imagine if this was working proberly itd be even better\n",
    "\n",
    "\n",
    "cm = confusion_matrix(y_train, Spam_model.predict(ngram_counts))\n",
    "print('Train data matrix: \\n', cm)\n",
    "cm = confusion_matrix(y_test, pred)\n",
    "print('Test data matrix: \\n', cm)\n",
    "\n",
    "plt.matshow(cm)\n",
    "plt.title('Confusion matrix')\n",
    "plt.suptitle('0=online, 1=lab')\n",
    "plt.colorbar()\n",
    "plt.ylabel('True label')\n",
    "plt.xlabel('Predicted label')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Try HMM\n",
    "\n",
    "Will have a generic attempt of implementing a Hidden Markov Model here, then in the next notebooks we can create relevant ones.\n",
    "Not entirely sure how these work but will word it out as I go along. \n",
    "\n",
    "This article is more complicated, but also indepth and explains stuff more. - https://towardsdatascience.com/markov-chains-and-hmms-ceaf2c854788\n",
    "\n",
    "Some guys github repository using HMM for binary sentimental analysis of tweets - https://github.com/FantacherJOY/Hidden-Markov-Model-for-NLP\n",
    "\n",
    "\"In the case one label is way more frequent than the other (say Class B appears for every 100 samples of Class A), or in the case one category is broader than the other (have more variance, i.e. Class A is an expected behavior of a process and Class B is all the cases that are abnormal), then we might want to train a unique HMM for Class A. For each new sample, we compute the likelihood to the model, and if it falls below a pre-defined threshold (that can be chosen with respect to some validation sample, or in the worst case, from the training samples themselves) we assign this sample to Class B. If the likelihood is above the threshold, we assign the sample to Class A.\" - https://datascience.stackexchange.com/questions/8560/can-hmm-be-used-as-a-binary-classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://hmmlearn.readthedocs.io/en/latest/tutorial.html\n",
    "# experimenting with the Working with multiple sequences part.]\n",
    "\n",
    "#https://stackoverflow.com/questions/52515134/hmmlearn-classification-usage\n",
    "\n",
    "from hmmlearn import hmm\n",
    "\n",
    "X1 = [[0.5], [1.0], [-1.0], [0.42], [0.24]]\n",
    "X2 = [[2.4], [4.2], [0.5], [-0.24]]\n",
    "X = np.concatenate([X1, X2]) # X = 1D array of length 9\n",
    "lengths = [len(X1), len(X2)] # lengths = [5,4], array of length 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = hmm.GaussianHMM(n_components=3).fit(X, lengths)\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 'The HMM is a generative probabilistic model, in which \n",
    "# a sequence of observable X variables is generated by a sequence of internal hidden states Z.\n",
    "# The hidden states are not observed directly.'\n",
    "\n",
    "X, Z = model.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now compare the models transisiton matrix to the actual one.\n",
    "# Might want this at the beginning to feed into model.\n",
    "# https://stackoverflow.com/questions/46657221/generating-markov-transition-matrix-in-python\n",
    "\n",
    "#the following code takes a list such as\n",
    "#[1,1,2,6,8,5,5,7,8,8,1,1,4,5,5,0,0,0,1,1,4,4,5,1,3,3,4,5,4,1,1]\n",
    "#with states labeled as successive integers starting with 0\n",
    "#and returns a transition matrix, M,\n",
    "#where M[i][j] is the probability of transitioning from i to j\n",
    "\n",
    "def transition_matrix(transitions):\n",
    "    n = 1+ max(transitions) #number of states\n",
    "\n",
    "    M = [[0]*n for _ in range(n)]\n",
    "\n",
    "    for (i,j) in zip(transitions,transitions[1:]):\n",
    "        M[i][j] += 1\n",
    "\n",
    "    #now convert to probabilities:\n",
    "    for row in M:\n",
    "        s = sum(row)\n",
    "        if s > 0:\n",
    "            row[:] = [f/s for f in row]\n",
    "    return M\n",
    "\n",
    "#test:\n",
    "transition_matrix(np.array(X).flatten())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Practise HMM Done\n",
    "\n",
    "Finished the initial exploration as to how the HMM implementation in python works.\n",
    "Now try to apply it to my data."
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3.7.6 64-bit ('Anaconda3': conda)",
   "language": "python",
   "name": "python37664bitanaconda3conda869ee3b15737460b8abcd30240fba0c8"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}