Next_word_Prediction_Python.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Next word Prediction Python.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyO0vERpe+5INQCLPVE+XI+F",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/vsnupoudel/data-science-capstone-JohnsHopkins/blob/main/Next_word_Prediction_Python.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nOoJJ05CcVjc"
      },
      "source": [
        "Download the text files and the Embedding file"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jly78TXbcZfh",
        "outputId": "b66f5dae-288e-43b2-fd5b-439017f50be6"
      },
      "source": [
        "!gdown --id 1hlvOXJqe3fdxjQilNfLRwPud_KWcOVEE -O '/content/en_US.zip'"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1hlvOXJqe3fdxjQilNfLRwPud_KWcOVEE\n",
            "To: /content/en_US.zip\n",
            "262MB [00:02, 95.9MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EoxZk98icm0z",
        "outputId": "3bc7c71b-4a1b-4194-dc1d-7d256d66532f"
      },
      "source": [
        "!unzip -o en_US.zip "
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  en_US.zip\n",
            "  inflating: en_US/en_US.blogs.txt   \n",
            "  inflating: en_US/en_US.news.txt    \n",
            "  inflating: en_US/en_US.twitter.txt  \n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8wwWG4ywdvER",
        "outputId": "d6d7e6b0-b133-4907-b13d-08f2a1987661"
      },
      "source": [
        " !gdown --id 1r3MHZdvnlC_BG8z6Jr1td-zJTC2GdqYK -O 'glove.6B.50d.txt' "
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading...\n",
            "From: https://drive.google.com/uc?id=1r3MHZdvnlC_BG8z6Jr1td-zJTC2GdqYK\n",
            "To: /content/glove.6B.50d.txt\n",
            "171MB [00:01, 92.4MB/s]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8JtXXgwRpTKq"
      },
      "source": [
        "# # establish connections with three files \n",
        "with open('./en_US/en_US.blogs.txt','r') as f:\n",
        "  blogs= f.readlines(99999)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XCNL7F2QvVXl"
      },
      "source": [
        "References :\n",
        "\n",
        "\n",
        " [Towards Data Science Article](https://towardsdatascience.com/exploring-the-next-word-predictor-5e22aeb85d8f)\n",
        "\n",
        "[Machine Learning Mastery](https://machinelearningmastery.com/use-word-embedding-layers-deep-learning-keras/)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KNa3bMDZUb_l"
      },
      "source": [
        "string = ' '.join(blogs) \n",
        "del blogs"
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2_PmoDGupTay"
      },
      "source": [
        "from keras.preprocessing.text import Tokenizer\n",
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "import numpy as np\n",
        "import re\n",
        "from keras.utils import to_categorical"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Jb49Std7z2Nk",
        "outputId": "78c7a5b2-76ac-44b2-ad93-6394a243d38e"
      },
      "source": [
        "nltk.download('punkt')"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jJAH7pKnqKPv"
      },
      "source": [
        "Removing non word characters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SwpiBsdEpiUe"
      },
      "source": [
        "string = re.sub(pattern='\\W+', repl=\" \", string = string ).lower()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nsJXJLtmzc1a"
      },
      "source": [
        "Tokenize and generate ngrams"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pFRmyvG5tO42"
      },
      "source": [
        "tokens = word_tokenize(string)\n",
        "train_len = 3\n",
        "text_sequences = []\n",
        "\n",
        "for i in range(train_len,len(tokens)):\n",
        "  seq = tokens[i-train_len:i]\n",
        "  text_sequences.append(seq)"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "857n8SrgAq8a"
      },
      "source": [
        "Read in 50 dimensional embeddings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 126
        },
        "id": "xduyO2LPApt2",
        "outputId": "40b95f4b-6620-4c89-ace0-3a9decd67ecb"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "colnames = ['word']+[ 'dim_'+str(i) for i in range(1,51)]\n",
        "Embed50 = pd.read_csv('glove.6B.50d.txt', engine='python'\n",
        "                  ,header=None, delim_whitespace=True ,\n",
        "                  names = colnames)\n",
        "Embed50.head(2)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>word</th>\n",
              "      <th>dim_1</th>\n",
              "      <th>dim_2</th>\n",
              "      <th>dim_3</th>\n",
              "      <th>dim_4</th>\n",
              "      <th>dim_5</th>\n",
              "      <th>dim_6</th>\n",
              "      <th>dim_7</th>\n",
              "      <th>dim_8</th>\n",
              "      <th>dim_9</th>\n",
              "      <th>dim_10</th>\n",
              "      <th>dim_11</th>\n",
              "      <th>dim_12</th>\n",
              "      <th>dim_13</th>\n",
              "      <th>dim_14</th>\n",
              "      <th>dim_15</th>\n",
              "      <th>dim_16</th>\n",
              "      <th>dim_17</th>\n",
              "      <th>dim_18</th>\n",
              "      <th>dim_19</th>\n",
              "      <th>dim_20</th>\n",
              "      <th>dim_21</th>\n",
              "      <th>dim_22</th>\n",
              "      <th>dim_23</th>\n",
              "      <th>dim_24</th>\n",
              "      <th>dim_25</th>\n",
              "      <th>dim_26</th>\n",
              "      <th>dim_27</th>\n",
              "      <th>dim_28</th>\n",
              "      <th>dim_29</th>\n",
              "      <th>dim_30</th>\n",
              "      <th>dim_31</th>\n",
              "      <th>dim_32</th>\n",
              "      <th>dim_33</th>\n",
              "      <th>dim_34</th>\n",
              "      <th>dim_35</th>\n",
              "      <th>dim_36</th>\n",
              "      <th>dim_37</th>\n",
              "      <th>dim_38</th>\n",
              "      <th>dim_39</th>\n",
              "      <th>dim_40</th>\n",
              "      <th>dim_41</th>\n",
              "      <th>dim_42</th>\n",
              "      <th>dim_43</th>\n",
              "      <th>dim_44</th>\n",
              "      <th>dim_45</th>\n",
              "      <th>dim_46</th>\n",
              "      <th>dim_47</th>\n",
              "      <th>dim_48</th>\n",
              "      <th>dim_49</th>\n",
              "      <th>dim_50</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>the</td>\n",
              "      <td>0.418000</td>\n",
              "      <td>0.24968</td>\n",
              "      <td>-0.41242</td>\n",
              "      <td>0.12170</td>\n",
              "      <td>0.34527</td>\n",
              "      <td>-0.044457</td>\n",
              "      <td>-0.49688</td>\n",
              "      <td>-0.17862</td>\n",
              "      <td>-0.00066</td>\n",
              "      <td>-0.65660</td>\n",
              "      <td>0.27843</td>\n",
              "      <td>-0.147670</td>\n",
              "      <td>-0.55677</td>\n",
              "      <td>0.14658</td>\n",
              "      <td>-0.00951</td>\n",
              "      <td>0.011658</td>\n",
              "      <td>0.10204</td>\n",
              "      <td>-0.127920</td>\n",
              "      <td>-0.84430</td>\n",
              "      <td>-0.12181</td>\n",
              "      <td>-0.016801</td>\n",
              "      <td>-0.33279</td>\n",
              "      <td>-0.15520</td>\n",
              "      <td>-0.23131</td>\n",
              "      <td>-0.191810</td>\n",
              "      <td>-1.8823</td>\n",
              "      <td>-0.76746</td>\n",
              "      <td>0.099051</td>\n",
              "      <td>-0.421250</td>\n",
              "      <td>-0.19526</td>\n",
              "      <td>4.0071</td>\n",
              "      <td>-0.18594</td>\n",
              "      <td>-0.522870</td>\n",
              "      <td>-0.31681</td>\n",
              "      <td>0.000592</td>\n",
              "      <td>0.007445</td>\n",
              "      <td>0.17778</td>\n",
              "      <td>-0.15897</td>\n",
              "      <td>0.012041</td>\n",
              "      <td>-0.054223</td>\n",
              "      <td>-0.298710</td>\n",
              "      <td>-0.15749</td>\n",
              "      <td>-0.34758</td>\n",
              "      <td>-0.045637</td>\n",
              "      <td>-0.44251</td>\n",
              "      <td>0.18785</td>\n",
              "      <td>0.002785</td>\n",
              "      <td>-0.18411</td>\n",
              "      <td>-0.115140</td>\n",
              "      <td>-0.78581</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>,</td>\n",
              "      <td>0.013441</td>\n",
              "      <td>0.23682</td>\n",
              "      <td>-0.16899</td>\n",
              "      <td>0.40951</td>\n",
              "      <td>0.63812</td>\n",
              "      <td>0.477090</td>\n",
              "      <td>-0.42852</td>\n",
              "      <td>-0.55641</td>\n",
              "      <td>-0.36400</td>\n",
              "      <td>-0.23938</td>\n",
              "      <td>0.13001</td>\n",
              "      <td>-0.063734</td>\n",
              "      <td>-0.39575</td>\n",
              "      <td>-0.48162</td>\n",
              "      <td>0.23291</td>\n",
              "      <td>0.090201</td>\n",
              "      <td>-0.13324</td>\n",
              "      <td>0.078639</td>\n",
              "      <td>-0.41634</td>\n",
              "      <td>-0.15428</td>\n",
              "      <td>0.100680</td>\n",
              "      <td>0.48891</td>\n",
              "      <td>0.31226</td>\n",
              "      <td>-0.12520</td>\n",
              "      <td>-0.037512</td>\n",
              "      <td>-1.5179</td>\n",
              "      <td>0.12612</td>\n",
              "      <td>-0.024420</td>\n",
              "      <td>-0.042961</td>\n",
              "      <td>-0.28351</td>\n",
              "      <td>3.5416</td>\n",
              "      <td>-0.11956</td>\n",
              "      <td>-0.014533</td>\n",
              "      <td>-0.14990</td>\n",
              "      <td>0.218640</td>\n",
              "      <td>-0.334120</td>\n",
              "      <td>-0.13872</td>\n",
              "      <td>0.31806</td>\n",
              "      <td>0.703580</td>\n",
              "      <td>0.448580</td>\n",
              "      <td>-0.080262</td>\n",
              "      <td>0.63003</td>\n",
              "      <td>0.32111</td>\n",
              "      <td>-0.467650</td>\n",
              "      <td>0.22786</td>\n",
              "      <td>0.36034</td>\n",
              "      <td>-0.378180</td>\n",
              "      <td>-0.56657</td>\n",
              "      <td>0.044691</td>\n",
              "      <td>0.30392</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "  word     dim_1    dim_2    dim_3  ...    dim_47   dim_48    dim_49   dim_50\n",
              "0  the  0.418000  0.24968 -0.41242  ...  0.002785 -0.18411 -0.115140 -0.78581\n",
              "1    ,  0.013441  0.23682 -0.16899  ... -0.378180 -0.56657  0.044691  0.30392\n",
              "\n",
              "[2 rows x 51 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kk79tGR4_NHF"
      },
      "source": [
        "Tokenizer class from keras.preprocessing.text"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ee8a7oiRUI79"
      },
      "source": [
        "del tokens\n",
        "del seq\n",
        "del string"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Snk535GPcwWE"
      },
      "source": [
        "Delete those rows which have words, that are not there in the pretrained embeddings. Found 100 such sequences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nLsp8HhuAo-J"
      },
      "source": [
        "embedlist = list(Embed50.word)\n",
        "delete_list = []\n",
        "for seq in text_sequences:\n",
        "  check =  all(item in embedlist for item in seq)\n",
        "  if not check:\n",
        "    delete_list.append(seq)\n",
        "\n",
        "text_sequences_del = [x for x in text_sequences if x not in delete_list]\n",
        "del text_sequences"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JmEqMAifzJcU"
      },
      "source": [
        "tokenizer = Tokenizer()\n",
        "tokenizer.fit_on_texts(text_sequences_del)\n",
        "sequences = tokenizer.texts_to_sequences(text_sequences_del)\n",
        "#vocabulary size increased by 1 \n",
        "vocabulary_size = len(tokenizer.word_counts)+1\n",
        "n_sequences = np.asarray ( sequences , dtype='int32')"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5kzTW4JyPio0"
      },
      "source": [
        "dic = pd.DataFrame( tokenizer.word_index.items(), columns= ['word','index'])"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PTIglsHUyyXZ",
        "outputId": "a042703f-6048-44ed-dca0-6225836aa3f0"
      },
      "source": [
        "dic.shape"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4492, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DRGHoxSGWzRE"
      },
      "source": [
        "word_embeds_pre = pd.merge(\n",
        "    left = dic,\n",
        "    right = Embed50 ,\n",
        "    how=\"inner\",\n",
        "    on=\"word\" ).iloc[: , 2:52] "
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZyCjVZquX6Gb",
        "outputId": "cef060e9-609b-4121-d26f-a69509117cfe"
      },
      "source": [
        "word_embeds_pre.shape"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(4492, 50)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t_PMKSWl3RQP"
      },
      "source": [
        "# X and Y\n",
        "train_inputs = n_sequences[:,:-1]\n",
        "train_targets = n_sequences[:,-1]\n",
        "train_targets = to_categorical(  train_targets \n",
        "                              , num_classes = vocabulary_size  )\n",
        "seq_len = train_inputs.shape[1]"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XVrlacSr0mmV",
        "outputId": "12008cd5-17df-44aa-d685-d31bf1500176"
      },
      "source": [
        "train_inputs.shape"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(18343, 2)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kneGfEEx0b6R",
        "outputId": "c2008878-d004-4057-f4e5-f175728282c0"
      },
      "source": [
        "train_targets.shape"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(18343, 4493)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5UW-dsz5G1f8"
      },
      "source": [
        "# Define model\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense\n",
        "from keras.layers import LSTM\n",
        "from keras.layers import Embedding\n",
        "model = Sequential()\n",
        "embed_layer = model.add(Embedding( input_dim = vocabulary_size-1\n",
        "                                  , output_dim = 50 \n",
        "                                  , input_length = seq_len\n",
        "                                  , weights = [word_embeds_pre]\n",
        "                                  , trainable = False\n",
        "                                   ) )\n",
        "model.add(LSTM(100,return_sequences=True))\n",
        "model.add(LSTM(50))\n",
        "model.add(Dense(100,activation='relu'))\n",
        "model.add(Dense(vocabulary_size, activation='softmax'))\n",
        "\n",
        "# compiling the network\n",
        "model.compile(loss='categorical_crossentropy', optimizer='adam'\n",
        ", metrics=['accuracy'])"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "C869lTHNgUfN",
        "outputId": "9aab3f4e-f50a-4d23-a824-634af7114614"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"sequential\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "embedding (Embedding)        (None, 2, 50)             224600    \n",
            "_________________________________________________________________\n",
            "lstm (LSTM)                  (None, 2, 100)            60400     \n",
            "_________________________________________________________________\n",
            "lstm_1 (LSTM)                (None, 50)                30200     \n",
            "_________________________________________________________________\n",
            "dense (Dense)                (None, 100)               5100      \n",
            "_________________________________________________________________\n",
            "dense_1 (Dense)              (None, 4493)              453793    \n",
            "=================================================================\n",
            "Total params: 774,093\n",
            "Trainable params: 549,493\n",
            "Non-trainable params: 224,600\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jGRxk3M2N48x",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ff8af839-8379-451a-9005-40eb508e1355"
      },
      "source": [
        "#fit model\n",
        "model.fit(train_inputs,train_targets,epochs=555,verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/555\n",
            "574/574 [==============================] - 37s 6ms/step - loss: 7.3726 - accuracy: 0.0410\n",
            "Epoch 2/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 6.6745 - accuracy: 0.0500\n",
            "Epoch 3/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 6.6968 - accuracy: 0.0473\n",
            "Epoch 4/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 6.6088 - accuracy: 0.0489\n",
            "Epoch 5/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 6.4817 - accuracy: 0.0521\n",
            "Epoch 6/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 6.3503 - accuracy: 0.0581\n",
            "Epoch 7/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 6.1236 - accuracy: 0.0659\n",
            "Epoch 8/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 5.9298 - accuracy: 0.0728\n",
            "Epoch 9/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 5.7270 - accuracy: 0.0737\n",
            "Epoch 10/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 5.5448 - accuracy: 0.0795\n",
            "Epoch 11/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 5.3373 - accuracy: 0.0852\n",
            "Epoch 12/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 5.1298 - accuracy: 0.0893\n",
            "Epoch 13/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 4.9318 - accuracy: 0.0906\n",
            "Epoch 14/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 4.7111 - accuracy: 0.1003\n",
            "Epoch 15/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 4.4742 - accuracy: 0.1135\n",
            "Epoch 16/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 4.2502 - accuracy: 0.1226\n",
            "Epoch 17/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 3.9915 - accuracy: 0.1548\n",
            "Epoch 18/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 3.7829 - accuracy: 0.1841\n",
            "Epoch 19/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 3.5590 - accuracy: 0.2254\n",
            "Epoch 20/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 3.3643 - accuracy: 0.2580\n",
            "Epoch 21/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 3.2126 - accuracy: 0.2804\n",
            "Epoch 22/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 3.0842 - accuracy: 0.3000\n",
            "Epoch 23/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.9667 - accuracy: 0.3165\n",
            "Epoch 24/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.8441 - accuracy: 0.3373\n",
            "Epoch 25/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.7459 - accuracy: 0.3500\n",
            "Epoch 26/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.6469 - accuracy: 0.3693\n",
            "Epoch 27/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.5912 - accuracy: 0.3761\n",
            "Epoch 28/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.5020 - accuracy: 0.3895\n",
            "Epoch 29/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.4338 - accuracy: 0.4020\n",
            "Epoch 30/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.3456 - accuracy: 0.4150\n",
            "Epoch 31/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.2926 - accuracy: 0.4284\n",
            "Epoch 32/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.2294 - accuracy: 0.4388\n",
            "Epoch 33/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.1810 - accuracy: 0.4491\n",
            "Epoch 34/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.1447 - accuracy: 0.4578\n",
            "Epoch 35/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.0740 - accuracy: 0.4702\n",
            "Epoch 36/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 2.0241 - accuracy: 0.4786\n",
            "Epoch 37/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.9847 - accuracy: 0.4874\n",
            "Epoch 38/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.9312 - accuracy: 0.4998\n",
            "Epoch 39/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.8977 - accuracy: 0.5041\n",
            "Epoch 40/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.8470 - accuracy: 0.5210\n",
            "Epoch 41/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.8068 - accuracy: 0.5240\n",
            "Epoch 42/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.7600 - accuracy: 0.5369\n",
            "Epoch 43/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.7282 - accuracy: 0.5457\n",
            "Epoch 44/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.6895 - accuracy: 0.5509\n",
            "Epoch 45/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.6642 - accuracy: 0.5553\n",
            "Epoch 46/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.6326 - accuracy: 0.5597\n",
            "Epoch 47/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.5819 - accuracy: 0.5753\n",
            "Epoch 48/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.5479 - accuracy: 0.5787\n",
            "Epoch 49/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.5284 - accuracy: 0.5840\n",
            "Epoch 50/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.5268 - accuracy: 0.5899\n",
            "Epoch 51/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.4618 - accuracy: 0.5980\n",
            "Epoch 52/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.4547 - accuracy: 0.5972\n",
            "Epoch 53/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.3978 - accuracy: 0.6168\n",
            "Epoch 54/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.3901 - accuracy: 0.6171\n",
            "Epoch 55/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.3631 - accuracy: 0.6233\n",
            "Epoch 56/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.3374 - accuracy: 0.6313\n",
            "Epoch 57/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.3070 - accuracy: 0.6349\n",
            "Epoch 58/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.2775 - accuracy: 0.6463\n",
            "Epoch 59/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.2439 - accuracy: 0.6523\n",
            "Epoch 60/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.2333 - accuracy: 0.6523\n",
            "Epoch 61/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.2209 - accuracy: 0.6546\n",
            "Epoch 62/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.1979 - accuracy: 0.6617\n",
            "Epoch 63/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.1648 - accuracy: 0.6682\n",
            "Epoch 64/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.1400 - accuracy: 0.6717\n",
            "Epoch 65/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.1472 - accuracy: 0.6749\n",
            "Epoch 66/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.1141 - accuracy: 0.6815\n",
            "Epoch 67/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.1171 - accuracy: 0.6768\n",
            "Epoch 68/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.0965 - accuracy: 0.6824\n",
            "Epoch 69/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.0666 - accuracy: 0.6907\n",
            "Epoch 70/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.0386 - accuracy: 0.7015\n",
            "Epoch 71/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.0309 - accuracy: 0.7030\n",
            "Epoch 72/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.0133 - accuracy: 0.7007\n",
            "Epoch 73/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 1.0081 - accuracy: 0.7063\n",
            "Epoch 74/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9832 - accuracy: 0.7067\n",
            "Epoch 75/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9728 - accuracy: 0.7117\n",
            "Epoch 76/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9418 - accuracy: 0.7159\n",
            "Epoch 77/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9616 - accuracy: 0.7163\n",
            "Epoch 78/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9284 - accuracy: 0.7168\n",
            "Epoch 79/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9150 - accuracy: 0.7264\n",
            "Epoch 80/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.9096 - accuracy: 0.7240\n",
            "Epoch 81/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8941 - accuracy: 0.7304\n",
            "Epoch 82/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8782 - accuracy: 0.7344\n",
            "Epoch 83/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8854 - accuracy: 0.7241\n",
            "Epoch 84/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8627 - accuracy: 0.7359\n",
            "Epoch 85/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8605 - accuracy: 0.7334\n",
            "Epoch 86/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8394 - accuracy: 0.7425\n",
            "Epoch 87/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8276 - accuracy: 0.7468\n",
            "Epoch 88/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8370 - accuracy: 0.7380\n",
            "Epoch 89/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8159 - accuracy: 0.7442\n",
            "Epoch 90/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8297 - accuracy: 0.7423\n",
            "Epoch 91/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8030 - accuracy: 0.7501\n",
            "Epoch 92/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8095 - accuracy: 0.7441\n",
            "Epoch 93/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7872 - accuracy: 0.7524\n",
            "Epoch 94/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7880 - accuracy: 0.7506\n",
            "Epoch 95/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.8038 - accuracy: 0.7467\n",
            "Epoch 96/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7688 - accuracy: 0.7557\n",
            "Epoch 97/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7700 - accuracy: 0.7508\n",
            "Epoch 98/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7698 - accuracy: 0.7512\n",
            "Epoch 99/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7622 - accuracy: 0.7557\n",
            "Epoch 100/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7621 - accuracy: 0.7520\n",
            "Epoch 101/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7557 - accuracy: 0.7519\n",
            "Epoch 102/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7455 - accuracy: 0.7540\n",
            "Epoch 103/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7418 - accuracy: 0.7582\n",
            "Epoch 104/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7404 - accuracy: 0.7533\n",
            "Epoch 105/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7507 - accuracy: 0.7515\n",
            "Epoch 106/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7189 - accuracy: 0.7626\n",
            "Epoch 107/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7260 - accuracy: 0.7545\n",
            "Epoch 108/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7288 - accuracy: 0.7597\n",
            "Epoch 109/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7225 - accuracy: 0.7557\n",
            "Epoch 110/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7138 - accuracy: 0.7571\n",
            "Epoch 111/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7192 - accuracy: 0.7583\n",
            "Epoch 112/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.7162 - accuracy: 0.7577\n",
            "Epoch 113/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7099 - accuracy: 0.7624\n",
            "Epoch 114/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6925 - accuracy: 0.7647\n",
            "Epoch 115/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.7047 - accuracy: 0.7618\n",
            "Epoch 116/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.7100 - accuracy: 0.7591\n",
            "Epoch 117/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6904 - accuracy: 0.7635\n",
            "Epoch 118/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.7062 - accuracy: 0.7568\n",
            "Epoch 119/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.7042 - accuracy: 0.7582\n",
            "Epoch 120/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6942 - accuracy: 0.7582\n",
            "Epoch 121/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6777 - accuracy: 0.7698\n",
            "Epoch 122/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6943 - accuracy: 0.7623\n",
            "Epoch 123/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6807 - accuracy: 0.7622\n",
            "Epoch 124/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6837 - accuracy: 0.7593\n",
            "Epoch 125/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6697 - accuracy: 0.7697\n",
            "Epoch 126/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6711 - accuracy: 0.7640\n",
            "Epoch 127/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6946 - accuracy: 0.7615\n",
            "Epoch 128/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6583 - accuracy: 0.7667\n",
            "Epoch 129/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6605 - accuracy: 0.7712\n",
            "Epoch 130/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6659 - accuracy: 0.7656\n",
            "Epoch 131/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6655 - accuracy: 0.7639\n",
            "Epoch 132/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6731 - accuracy: 0.7606\n",
            "Epoch 133/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6713 - accuracy: 0.7597\n",
            "Epoch 134/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6545 - accuracy: 0.7675\n",
            "Epoch 135/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6627 - accuracy: 0.7649\n",
            "Epoch 136/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6579 - accuracy: 0.7688\n",
            "Epoch 137/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6488 - accuracy: 0.7661\n",
            "Epoch 138/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6530 - accuracy: 0.7654\n",
            "Epoch 139/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6620 - accuracy: 0.7663\n",
            "Epoch 140/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6508 - accuracy: 0.7681\n",
            "Epoch 141/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6576 - accuracy: 0.7636\n",
            "Epoch 142/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6488 - accuracy: 0.7685\n",
            "Epoch 143/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6382 - accuracy: 0.7737\n",
            "Epoch 144/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6480 - accuracy: 0.7643\n",
            "Epoch 145/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6372 - accuracy: 0.7706\n",
            "Epoch 146/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6419 - accuracy: 0.7661\n",
            "Epoch 147/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6409 - accuracy: 0.7679\n",
            "Epoch 148/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6610 - accuracy: 0.7631\n",
            "Epoch 149/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6364 - accuracy: 0.7671\n",
            "Epoch 150/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6395 - accuracy: 0.7655\n",
            "Epoch 151/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6335 - accuracy: 0.7709\n",
            "Epoch 152/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6482 - accuracy: 0.7606\n",
            "Epoch 153/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6392 - accuracy: 0.7663\n",
            "Epoch 154/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6277 - accuracy: 0.7683\n",
            "Epoch 155/555\n",
            "574/574 [==============================] - 3s 5ms/step - loss: 0.6309 - accuracy: 0.7687\n",
            "Epoch 156/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6262 - accuracy: 0.7689\n",
            "Epoch 157/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6364 - accuracy: 0.7669\n",
            "Epoch 158/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6360 - accuracy: 0.7603\n",
            "Epoch 159/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6274 - accuracy: 0.7719\n",
            "Epoch 160/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6306 - accuracy: 0.7651\n",
            "Epoch 161/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6334 - accuracy: 0.7663\n",
            "Epoch 162/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6385 - accuracy: 0.7659\n",
            "Epoch 163/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6343 - accuracy: 0.7619\n",
            "Epoch 164/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6224 - accuracy: 0.7640\n",
            "Epoch 165/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6355 - accuracy: 0.7644\n",
            "Epoch 166/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6490 - accuracy: 0.7570\n",
            "Epoch 167/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6197 - accuracy: 0.7665\n",
            "Epoch 168/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6315 - accuracy: 0.7627\n",
            "Epoch 169/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6223 - accuracy: 0.7671\n",
            "Epoch 170/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6121 - accuracy: 0.7708\n",
            "Epoch 171/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6123 - accuracy: 0.7696\n",
            "Epoch 172/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6192 - accuracy: 0.7624\n",
            "Epoch 173/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6096 - accuracy: 0.7706\n",
            "Epoch 174/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6185 - accuracy: 0.7695\n",
            "Epoch 175/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6272 - accuracy: 0.7632\n",
            "Epoch 176/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6110 - accuracy: 0.7653\n",
            "Epoch 177/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6095 - accuracy: 0.7688\n",
            "Epoch 178/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6101 - accuracy: 0.7707\n",
            "Epoch 179/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6323 - accuracy: 0.7625\n",
            "Epoch 180/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6144 - accuracy: 0.7642\n",
            "Epoch 181/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6253 - accuracy: 0.7657\n",
            "Epoch 182/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6190 - accuracy: 0.7657\n",
            "Epoch 183/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6049 - accuracy: 0.7657\n",
            "Epoch 184/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6187 - accuracy: 0.7663\n",
            "Epoch 185/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6311 - accuracy: 0.7590\n",
            "Epoch 186/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6188 - accuracy: 0.7643\n",
            "Epoch 187/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6072 - accuracy: 0.7670\n",
            "Epoch 188/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6093 - accuracy: 0.7688\n",
            "Epoch 189/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6199 - accuracy: 0.7678\n",
            "Epoch 190/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6070 - accuracy: 0.7688\n",
            "Epoch 191/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5961 - accuracy: 0.7714\n",
            "Epoch 192/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6158 - accuracy: 0.7683\n",
            "Epoch 193/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5929 - accuracy: 0.7692\n",
            "Epoch 194/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6103 - accuracy: 0.7656\n",
            "Epoch 195/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6011 - accuracy: 0.7678\n",
            "Epoch 196/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6114 - accuracy: 0.7678\n",
            "Epoch 197/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5972 - accuracy: 0.7715\n",
            "Epoch 198/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5979 - accuracy: 0.7676\n",
            "Epoch 199/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5972 - accuracy: 0.7693\n",
            "Epoch 200/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5972 - accuracy: 0.7660\n",
            "Epoch 201/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6068 - accuracy: 0.7661\n",
            "Epoch 202/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5990 - accuracy: 0.7694\n",
            "Epoch 203/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6017 - accuracy: 0.7656\n",
            "Epoch 204/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6074 - accuracy: 0.7617\n",
            "Epoch 205/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6036 - accuracy: 0.7646\n",
            "Epoch 206/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6145 - accuracy: 0.7613\n",
            "Epoch 207/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5976 - accuracy: 0.7699\n",
            "Epoch 208/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6167 - accuracy: 0.7601\n",
            "Epoch 209/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5812 - accuracy: 0.7723\n",
            "Epoch 210/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6058 - accuracy: 0.7683\n",
            "Epoch 211/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5922 - accuracy: 0.7697\n",
            "Epoch 212/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5930 - accuracy: 0.7692\n",
            "Epoch 213/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5756 - accuracy: 0.7729\n",
            "Epoch 214/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6076 - accuracy: 0.7620\n",
            "Epoch 215/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5860 - accuracy: 0.7708\n",
            "Epoch 216/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5843 - accuracy: 0.7724\n",
            "Epoch 217/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5990 - accuracy: 0.7675\n",
            "Epoch 218/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5979 - accuracy: 0.7700\n",
            "Epoch 219/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6051 - accuracy: 0.7651\n",
            "Epoch 220/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5986 - accuracy: 0.7637\n",
            "Epoch 221/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5993 - accuracy: 0.7636\n",
            "Epoch 222/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5894 - accuracy: 0.7684\n",
            "Epoch 223/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5990 - accuracy: 0.7657\n",
            "Epoch 224/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5851 - accuracy: 0.7742\n",
            "Epoch 225/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5878 - accuracy: 0.7688\n",
            "Epoch 226/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5721 - accuracy: 0.7770\n",
            "Epoch 227/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5880 - accuracy: 0.7725\n",
            "Epoch 228/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5851 - accuracy: 0.7675\n",
            "Epoch 229/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5894 - accuracy: 0.7714\n",
            "Epoch 230/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5885 - accuracy: 0.7704\n",
            "Epoch 231/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6063 - accuracy: 0.7607\n",
            "Epoch 232/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5807 - accuracy: 0.7708\n",
            "Epoch 233/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5770 - accuracy: 0.7720\n",
            "Epoch 234/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5831 - accuracy: 0.7693\n",
            "Epoch 235/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5805 - accuracy: 0.7709\n",
            "Epoch 236/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.6094 - accuracy: 0.7631\n",
            "Epoch 237/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5900 - accuracy: 0.7660\n",
            "Epoch 238/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5853 - accuracy: 0.7677\n",
            "Epoch 239/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5900 - accuracy: 0.7642\n",
            "Epoch 240/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5879 - accuracy: 0.7692\n",
            "Epoch 241/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5882 - accuracy: 0.7662\n",
            "Epoch 242/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5915 - accuracy: 0.7666\n",
            "Epoch 243/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5923 - accuracy: 0.7672\n",
            "Epoch 244/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5839 - accuracy: 0.7660\n",
            "Epoch 245/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5815 - accuracy: 0.7707\n",
            "Epoch 246/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5963 - accuracy: 0.7656\n",
            "Epoch 247/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5801 - accuracy: 0.7686\n",
            "Epoch 248/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5849 - accuracy: 0.7662\n",
            "Epoch 249/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5876 - accuracy: 0.7649\n",
            "Epoch 250/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5770 - accuracy: 0.7714\n",
            "Epoch 251/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5800 - accuracy: 0.7700\n",
            "Epoch 252/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5847 - accuracy: 0.7675\n",
            "Epoch 253/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5815 - accuracy: 0.7679\n",
            "Epoch 254/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5833 - accuracy: 0.7678\n",
            "Epoch 255/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5846 - accuracy: 0.7647\n",
            "Epoch 256/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5744 - accuracy: 0.7663\n",
            "Epoch 257/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5815 - accuracy: 0.7688\n",
            "Epoch 258/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5838 - accuracy: 0.7682\n",
            "Epoch 259/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5745 - accuracy: 0.7694\n",
            "Epoch 260/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5898 - accuracy: 0.7633\n",
            "Epoch 261/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5829 - accuracy: 0.7636\n",
            "Epoch 262/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5787 - accuracy: 0.7681\n",
            "Epoch 263/555\n",
            "574/574 [==============================] - 3s 6ms/step - loss: 0.5832 - accuracy: 0.7679\n",
            "Epoch 264/555\n",
            "260/574 [============>.................] - ETA: 1s - loss: 0.5633 - accuracy: 0.7810"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U9dxx8dV5S67"
      },
      "source": [
        "train_targets.shape"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L5f31a7LPFwo"
      },
      "source": [
        "Predicting:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DWdSWUd6OCDz"
      },
      "source": [
        "from keras.preprocessing.sequence import pad_sequences\n",
        "\n",
        "input_text = input().strip().lower()\n",
        "encoded_text = tokenizer.texts_to_sequences([input_text])[0]\n",
        "pad_encoded = pad_sequences([encoded_text], maxlen=seq_len, truncating='pre')\n",
        "print('input: ', input_text, '\\nencoded text: ', encoded_text\n",
        "      , '\\n pad encoded: ' , pad_encoded)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8qhm5_5aPEml"
      },
      "source": [
        "for i in (model.predict(pad_encoded)[0]).argsort()[-3:][::-1]:\n",
        "  pred_word = tokenizer.index_word[i]\n",
        "  print(\"Next word suggestion:\",pred_word)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "I5sJRh4YPOxk"
      },
      "source": [
        "# type( model.predict(pad_encoded)[0].argsort( )[-3:][::-1] )"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}