R_for_Capstone.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "R for Capstone.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyNCj3SxaNS+zUUhgvU49ZSn",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "R",
      "name": "ir"
    },
    "language_info": {
      "name": "R"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/vsnupoudel/data-science-capstone-JohnsHopkins/blob/main/R_for_Capstone.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2cfHpbP7F0md"
      },
      "source": [
        "## After setting working directory"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "0D6gH9nuF_-a",
        "outputId": "214733bd-4509-449d-cd39-a4b68f59de9b"
      },
      "source": [
        "getwd()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "[1] \"/content\""
            ],
            "text/latex": "'/content'",
            "text/markdown": "'/content'",
            "text/html": [
              "'/content'"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Q8YLhC_RwMAO",
        "outputId": "7a53e0f1-7c97-4c49-a82f-e2f9a86e3a1e"
      },
      "source": [
        "# install.packages( c( \"tm\",\"ngram\",\"dplyr\",\"tidyr\",\"keras\",\"data.table\",\"stringr\") )\n",
        "x <- c( \"tm\", \"ngram\", \"dplyr\", \"tidyr\", \"keras\", \"data.table\")\n",
        "z = lapply(x, require, character.only = TRUE)"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading required package: tm\n",
            "\n",
            "Loading required package: NLP\n",
            "\n",
            "Loading required package: ngram\n",
            "\n",
            "Loading required package: dplyr\n",
            "\n",
            "\n",
            "Attaching package: ‘dplyr’\n",
            "\n",
            "\n",
            "The following objects are masked from ‘package:stats’:\n",
            "\n",
            "    filter, lag\n",
            "\n",
            "\n",
            "The following objects are masked from ‘package:base’:\n",
            "\n",
            "    intersect, setdiff, setequal, union\n",
            "\n",
            "\n",
            "Loading required package: tidyr\n",
            "\n",
            "Loading required package: keras\n",
            "\n",
            "Loading required package: data.table\n",
            "\n",
            "\n",
            "Attaching package: ‘data.table’\n",
            "\n",
            "\n",
            "The following objects are masked from ‘package:dplyr’:\n",
            "\n",
            "    between, first, last\n",
            "\n",
            "\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0jdwfAqqrA3g"
      },
      "source": [
        "system(\"gdown --id 1hlvOXJqe3fdxjQilNfLRwPud_KWcOVEE -O '/content/en_US.zip' \")"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lKzHv9DkxXHp"
      },
      "source": [
        "unzip(zipfile= '/content/en_US.zip', overwrite = TRUE )"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1-myLUobHnU-"
      },
      "source": [
        "system(\" gdown --id 1r3MHZdvnlC_BG8z6Jr1td-zJTC2GdqYK -O 'glove.6B.50d.txt'  \")"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ISklhRQgFxJ1",
        "outputId": "b3d51097-6097-4c6f-f998-44004963ce21"
      },
      "source": [
        "# # establish connections with three files \n",
        "print( getwd())\n",
        "con3 <- file( '/content/en_US/en_US.twitter.txt', 'r')\n",
        "text3 <- readLines(con3)\n",
        "close(con3)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1] \"/content\"\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Warning message in readLines(con3):\n",
            "“line 167155 appears to contain an embedded nul”\n",
            "Warning message in readLines(con3):\n",
            "“line 268547 appears to contain an embedded nul”\n",
            "Warning message in readLines(con3):\n",
            "“line 1274086 appears to contain an embedded nul”\n",
            "Warning message in readLines(con3):\n",
            "“line 1759032 appears to contain an embedded nul”\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sZOrFTcVL4Nx"
      },
      "source": [
        "# # sample 0.01% data\n",
        "sample_index <- sample(seq(text3), 0.01*length(text3))\n",
        "sample_text <- text3[sample_index]\n",
        "\n",
        "dir.create(file.path(\"./sample_text\"))\n",
        "con4 <- file('./sample_text/sampled_text.txt')\n",
        "writeLines(sample_text, con = con4)\n",
        "close(con4)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 392
        },
        "id": "hWIBvzbInbHT",
        "outputId": "b9f37c9d-b634-4432-83af-75dc1aff31af"
      },
      "source": [
        "# Using the tm library for preprocessing the sample text\n",
        "library(tm)\n",
        "data_folder = DirSource(directory = \"./sample_text\",\n",
        "            encoding = \"UTF-8\",\n",
        "            mode = \"text\")\n",
        "# load the file into Corpus object\n",
        "docs <- SimpleCorpus(data_folder )\n",
        "# Converting to lowercase:\n",
        "docs <- tm_map(docs, tolower)\n",
        "docs <- tm_map(docs, PlainTextDocument)\n",
        "\n",
        "\n",
        "library(ngram)\n",
        "string <- docs[[1]]$content\n",
        "# trying different libraries\n",
        "# library(tokenizers)\n",
        "# library(word2vec)\n",
        "# string.summary(string)\n",
        "ng <- ngram (string, n=3)\n",
        "# get.phrasetable(ng)\n",
        "# tokens <- tokenize_ngrams(string , n = 3 )\n",
        "\n",
        "# As a data frame\n",
        "df <- data.frame(get.phrasetable(ng))\n",
        "\n",
        "# Put the last word as target\n",
        "library(tidyr)\n",
        "sep<- separate(data = df, col = ngrams , into = c(\"X1\",\"X2\", \"y\")\n",
        "               # ,sep = \"^\\\\w+\\\\s+\"\n",
        "               # , remove=FALSE\n",
        "               )\n",
        "# Xy <- unite(sep , X , c(X1, X2 ), remove=TRUE, sep=\" \")\n",
        "df <- data.frame(sep)\n",
        "remove(sep)\n",
        "head(df)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning message in tm_map.SimpleCorpus(docs, PlainTextDocument):\n",
            "“transformation drops documents”\n",
            "Warning message:\n",
            "“Expected 3 pieces. Additional pieces discarded in 253991 rows [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, ...].”\n",
            "Warning message:\n",
            "“Expected 3 pieces. Missing pieces filled with `NA` in 99 rows [555, 2500, 2658, 5275, 9485, 13258, 16174, 19423, 20442, 23988, 26853, 32248, 32861, 35101, 45188, 47149, 53144, 56559, 61119, 63756, ...].”\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  X1     X2    y    freq prop        \n",
              "1 going  to    be   79   0.0002828561\n",
              "2 to     be    a    68   0.0002434710\n",
              "3 one    of    the  57   0.0002040860\n",
              "4 is     going to   55   0.0001969251\n",
              "5 thanks for   the  54   0.0001933446\n",
              "6 can    t     wait 51   0.0001826033"
            ],
            "text/latex": "A data.frame: 6 × 5\n\\begin{tabular}{r|lllll}\n  & X1 & X2 & y & freq & prop\\\\\n  & <chr> & <chr> & <chr> & <int> & <dbl>\\\\\n\\hline\n\t1 & going  & to    & be   & 79 & 0.0002828561\\\\\n\t2 & to     & be    & a    & 68 & 0.0002434710\\\\\n\t3 & one    & of    & the  & 57 & 0.0002040860\\\\\n\t4 & is     & going & to   & 55 & 0.0001969251\\\\\n\t5 & thanks & for   & the  & 54 & 0.0001933446\\\\\n\t6 & can    & t     & wait & 51 & 0.0001826033\\\\\n\\end{tabular}\n",
            "text/markdown": "\nA data.frame: 6 × 5\n\n| <!--/--> | X1 &lt;chr&gt; | X2 &lt;chr&gt; | y &lt;chr&gt; | freq &lt;int&gt; | prop &lt;dbl&gt; |\n|---|---|---|---|---|---|\n| 1 | going  | to    | be   | 79 | 0.0002828561 |\n| 2 | to     | be    | a    | 68 | 0.0002434710 |\n| 3 | one    | of    | the  | 57 | 0.0002040860 |\n| 4 | is     | going | to   | 55 | 0.0001969251 |\n| 5 | thanks | for   | the  | 54 | 0.0001933446 |\n| 6 | can    | t     | wait | 51 | 0.0001826033 |\n\n",
            "text/html": [
              "<table class=\"dataframe\">\n",
              "<caption>A data.frame: 6 × 5</caption>\n",
              "<thead>\n",
              "\t<tr><th></th><th scope=col>X1</th><th scope=col>X2</th><th scope=col>y</th><th scope=col>freq</th><th scope=col>prop</th></tr>\n",
              "\t<tr><th></th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;int&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "\t<tr><th scope=row>1</th><td>going </td><td>to   </td><td>be  </td><td>79</td><td>0.0002828561</td></tr>\n",
              "\t<tr><th scope=row>2</th><td>to    </td><td>be   </td><td>a   </td><td>68</td><td>0.0002434710</td></tr>\n",
              "\t<tr><th scope=row>3</th><td>one   </td><td>of   </td><td>the </td><td>57</td><td>0.0002040860</td></tr>\n",
              "\t<tr><th scope=row>4</th><td>is    </td><td>going</td><td>to  </td><td>55</td><td>0.0001969251</td></tr>\n",
              "\t<tr><th scope=row>5</th><td>thanks</td><td>for  </td><td>the </td><td>54</td><td>0.0001933446</td></tr>\n",
              "\t<tr><th scope=row>6</th><td>can   </td><td>t    </td><td>wait</td><td>51</td><td>0.0001826033</td></tr>\n",
              "</tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dfGi_wHdVFJu"
      },
      "source": [
        "References:\n",
        "\n",
        "[Keras Article](https://keras.rstudio.com/articles/examples/pretrained_word_embeddings.html)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 426
        },
        "id": "5x4FcDYvv8BU",
        "outputId": "e7700bc6-bc59-4ba4-e2ce-72eda8d3b82e"
      },
      "source": [
        "# Try downloading pre-trained word embeddings instead of training new\n",
        "# This example shows how one can quickly load glove vectors\n",
        "# and train a Keras model in R\n",
        "\n",
        "library(keras)\n",
        "library(dplyr)\n",
        "\n",
        "# Load the word embeddings as df \n",
        "# load glove vectors into R\n",
        "vectors = data.table::fread('glove.6B.50d.txt', data.table = F\n",
        "                            ,  encoding = 'UTF-8')\n",
        "colnames(vectors) = c('word',paste('dim',1:50,sep = '_'))\n",
        "\n",
        "vecDf <- as_tibble(vectors)\n",
        "head(vecDf)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Warning message in data.table::fread(\"glove.6B.50d.txt\", data.table = F, encoding = \"UTF-8\"):\n",
            "“Found and resolved improper quoting in first 100 rows. If the fields are not quoted (e.g. field separator does not appear within any field), try quote=\"\" to avoid this warning.”\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  word dim_1    dim_2     dim_3    dim_4     dim_5   dim_6     dim_7   \n",
              "1 the  0.418000  0.249680 -0.41242  0.121700 0.34527 -0.044457 -0.49688\n",
              "2 ,    0.013441  0.236820 -0.16899  0.409510 0.63812  0.477090 -0.42852\n",
              "3 .    0.151640  0.301770 -0.16763  0.176840 0.31719  0.339730 -0.43478\n",
              "4 of   0.708530  0.570880 -0.47160  0.180480 0.54449  0.726030  0.18157\n",
              "5 to   0.680470 -0.039263  0.30186 -0.177920 0.42962  0.032246 -0.41376\n",
              "6 and  0.268180  0.143460 -0.27877  0.016257 0.11384  0.699230 -0.51332\n",
              "  dim_8    dim_9       ⋯ dim_41      dim_42    dim_43    dim_44    dim_45  \n",
              "1 -0.17862 -0.00066023 ⋯ -2.9871e-01 -0.157490 -0.347580 -0.045637 -0.44251\n",
              "2 -0.55641 -0.36400000 ⋯ -8.0262e-02  0.630030  0.321110 -0.467650  0.22786\n",
              "3 -0.31086 -0.44999000 ⋯ -6.3681e-05  0.068987  0.087939 -0.102850 -0.13931\n",
              "4 -0.52393  0.10381000 ⋯ -3.4727e-01  0.284830  0.075693 -0.062178 -0.38988\n",
              "5  0.13228 -0.29847000 ⋯ -9.4375e-02  0.018324  0.210480 -0.030880 -0.19722\n",
              "6 -0.47368 -0.33075000 ⋯ -6.9043e-02  0.368850  0.251680 -0.245170  0.25381\n",
              "  dim_46   dim_47     dim_48    dim_49    dim_50  \n",
              "1 0.187850  0.0027849 -0.184110 -0.115140 -0.78581\n",
              "2 0.360340 -0.3781800 -0.566570  0.044691  0.30392\n",
              "3 0.223140 -0.0808030 -0.356520  0.016413  0.10216\n",
              "4 0.229020 -0.2161700 -0.225620 -0.093918 -0.80375\n",
              "5 0.082279 -0.0943400 -0.073297 -0.064699 -0.26044\n",
              "6 0.136700 -0.3117800 -0.632100 -0.250280 -0.38097"
            ],
            "text/latex": "A tibble: 6 × 51\n\\begin{tabular}{lllllllllllllllllllll}\n word & dim\\_1 & dim\\_2 & dim\\_3 & dim\\_4 & dim\\_5 & dim\\_6 & dim\\_7 & dim\\_8 & dim\\_9 & ⋯ & dim\\_41 & dim\\_42 & dim\\_43 & dim\\_44 & dim\\_45 & dim\\_46 & dim\\_47 & dim\\_48 & dim\\_49 & dim\\_50\\\\\n <chr> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & ⋯ & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl> & <dbl>\\\\\n\\hline\n\t the & 0.418000 &  0.249680 & -0.41242 &  0.121700 & 0.34527 & -0.044457 & -0.49688 & -0.17862 & -0.00066023 & ⋯ & -2.9871e-01 & -0.157490 & -0.347580 & -0.045637 & -0.44251 & 0.187850 &  0.0027849 & -0.184110 & -0.115140 & -0.78581\\\\\n\t ,   & 0.013441 &  0.236820 & -0.16899 &  0.409510 & 0.63812 &  0.477090 & -0.42852 & -0.55641 & -0.36400000 & ⋯ & -8.0262e-02 &  0.630030 &  0.321110 & -0.467650 &  0.22786 & 0.360340 & -0.3781800 & -0.566570 &  0.044691 &  0.30392\\\\\n\t .   & 0.151640 &  0.301770 & -0.16763 &  0.176840 & 0.31719 &  0.339730 & -0.43478 & -0.31086 & -0.44999000 & ⋯ & -6.3681e-05 &  0.068987 &  0.087939 & -0.102850 & -0.13931 & 0.223140 & -0.0808030 & -0.356520 &  0.016413 &  0.10216\\\\\n\t of  & 0.708530 &  0.570880 & -0.47160 &  0.180480 & 0.54449 &  0.726030 &  0.18157 & -0.52393 &  0.10381000 & ⋯ & -3.4727e-01 &  0.284830 &  0.075693 & -0.062178 & -0.38988 & 0.229020 & -0.2161700 & -0.225620 & -0.093918 & -0.80375\\\\\n\t to  & 0.680470 & -0.039263 &  0.30186 & -0.177920 & 0.42962 &  0.032246 & -0.41376 &  0.13228 & -0.29847000 & ⋯ & -9.4375e-02 &  0.018324 &  0.210480 & -0.030880 & -0.19722 & 0.082279 & -0.0943400 & -0.073297 & -0.064699 & -0.26044\\\\\n\t and & 0.268180 &  0.143460 & -0.27877 &  0.016257 & 0.11384 &  0.699230 & -0.51332 & -0.47368 & -0.33075000 & ⋯ & -6.9043e-02 &  0.368850 &  0.251680 & -0.245170 &  0.25381 & 0.136700 & -0.3117800 & -0.632100 & -0.250280 & -0.38097\\\\\n\\end{tabular}\n",
            "text/markdown": "\nA tibble: 6 × 51\n\n| word &lt;chr&gt; | dim_1 &lt;dbl&gt; | dim_2 &lt;dbl&gt; | dim_3 &lt;dbl&gt; | dim_4 &lt;dbl&gt; | dim_5 &lt;dbl&gt; | dim_6 &lt;dbl&gt; | dim_7 &lt;dbl&gt; | dim_8 &lt;dbl&gt; | dim_9 &lt;dbl&gt; | ⋯ ⋯ | dim_41 &lt;dbl&gt; | dim_42 &lt;dbl&gt; | dim_43 &lt;dbl&gt; | dim_44 &lt;dbl&gt; | dim_45 &lt;dbl&gt; | dim_46 &lt;dbl&gt; | dim_47 &lt;dbl&gt; | dim_48 &lt;dbl&gt; | dim_49 &lt;dbl&gt; | dim_50 &lt;dbl&gt; |\n|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n| the | 0.418000 |  0.249680 | -0.41242 |  0.121700 | 0.34527 | -0.044457 | -0.49688 | -0.17862 | -0.00066023 | ⋯ | -2.9871e-01 | -0.157490 | -0.347580 | -0.045637 | -0.44251 | 0.187850 |  0.0027849 | -0.184110 | -0.115140 | -0.78581 |\n| ,   | 0.013441 |  0.236820 | -0.16899 |  0.409510 | 0.63812 |  0.477090 | -0.42852 | -0.55641 | -0.36400000 | ⋯ | -8.0262e-02 |  0.630030 |  0.321110 | -0.467650 |  0.22786 | 0.360340 | -0.3781800 | -0.566570 |  0.044691 |  0.30392 |\n| .   | 0.151640 |  0.301770 | -0.16763 |  0.176840 | 0.31719 |  0.339730 | -0.43478 | -0.31086 | -0.44999000 | ⋯ | -6.3681e-05 |  0.068987 |  0.087939 | -0.102850 | -0.13931 | 0.223140 | -0.0808030 | -0.356520 |  0.016413 |  0.10216 |\n| of  | 0.708530 |  0.570880 | -0.47160 |  0.180480 | 0.54449 |  0.726030 |  0.18157 | -0.52393 |  0.10381000 | ⋯ | -3.4727e-01 |  0.284830 |  0.075693 | -0.062178 | -0.38988 | 0.229020 | -0.2161700 | -0.225620 | -0.093918 | -0.80375 |\n| to  | 0.680470 | -0.039263 |  0.30186 | -0.177920 | 0.42962 |  0.032246 | -0.41376 |  0.13228 | -0.29847000 | ⋯ | -9.4375e-02 |  0.018324 |  0.210480 | -0.030880 | -0.19722 | 0.082279 | -0.0943400 | -0.073297 | -0.064699 | -0.26044 |\n| and | 0.268180 |  0.143460 | -0.27877 |  0.016257 | 0.11384 |  0.699230 | -0.51332 | -0.47368 | -0.33075000 | ⋯ | -6.9043e-02 |  0.368850 |  0.251680 | -0.245170 |  0.25381 | 0.136700 | -0.3117800 | -0.632100 | -0.250280 | -0.38097 |\n\n",
            "text/html": [
              "<table class=\"dataframe\">\n",
              "<caption>A tibble: 6 × 51</caption>\n",
              "<thead>\n",
              "\t<tr><th scope=col>word</th><th scope=col>dim_1</th><th scope=col>dim_2</th><th scope=col>dim_3</th><th scope=col>dim_4</th><th scope=col>dim_5</th><th scope=col>dim_6</th><th scope=col>dim_7</th><th scope=col>dim_8</th><th scope=col>dim_9</th><th scope=col>⋯</th><th scope=col>dim_41</th><th scope=col>dim_42</th><th scope=col>dim_43</th><th scope=col>dim_44</th><th scope=col>dim_45</th><th scope=col>dim_46</th><th scope=col>dim_47</th><th scope=col>dim_48</th><th scope=col>dim_49</th><th scope=col>dim_50</th></tr>\n",
              "\t<tr><th scope=col>&lt;chr&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>⋯</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th><th scope=col>&lt;dbl&gt;</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "\t<tr><td>the</td><td>0.418000</td><td> 0.249680</td><td>-0.41242</td><td> 0.121700</td><td>0.34527</td><td>-0.044457</td><td>-0.49688</td><td>-0.17862</td><td>-0.00066023</td><td>⋯</td><td>-2.9871e-01</td><td>-0.157490</td><td>-0.347580</td><td>-0.045637</td><td>-0.44251</td><td>0.187850</td><td> 0.0027849</td><td>-0.184110</td><td>-0.115140</td><td>-0.78581</td></tr>\n",
              "\t<tr><td>,  </td><td>0.013441</td><td> 0.236820</td><td>-0.16899</td><td> 0.409510</td><td>0.63812</td><td> 0.477090</td><td>-0.42852</td><td>-0.55641</td><td>-0.36400000</td><td>⋯</td><td>-8.0262e-02</td><td> 0.630030</td><td> 0.321110</td><td>-0.467650</td><td> 0.22786</td><td>0.360340</td><td>-0.3781800</td><td>-0.566570</td><td> 0.044691</td><td> 0.30392</td></tr>\n",
              "\t<tr><td>.  </td><td>0.151640</td><td> 0.301770</td><td>-0.16763</td><td> 0.176840</td><td>0.31719</td><td> 0.339730</td><td>-0.43478</td><td>-0.31086</td><td>-0.44999000</td><td>⋯</td><td>-6.3681e-05</td><td> 0.068987</td><td> 0.087939</td><td>-0.102850</td><td>-0.13931</td><td>0.223140</td><td>-0.0808030</td><td>-0.356520</td><td> 0.016413</td><td> 0.10216</td></tr>\n",
              "\t<tr><td>of </td><td>0.708530</td><td> 0.570880</td><td>-0.47160</td><td> 0.180480</td><td>0.54449</td><td> 0.726030</td><td> 0.18157</td><td>-0.52393</td><td> 0.10381000</td><td>⋯</td><td>-3.4727e-01</td><td> 0.284830</td><td> 0.075693</td><td>-0.062178</td><td>-0.38988</td><td>0.229020</td><td>-0.2161700</td><td>-0.225620</td><td>-0.093918</td><td>-0.80375</td></tr>\n",
              "\t<tr><td>to </td><td>0.680470</td><td>-0.039263</td><td> 0.30186</td><td>-0.177920</td><td>0.42962</td><td> 0.032246</td><td>-0.41376</td><td> 0.13228</td><td>-0.29847000</td><td>⋯</td><td>-9.4375e-02</td><td> 0.018324</td><td> 0.210480</td><td>-0.030880</td><td>-0.19722</td><td>0.082279</td><td>-0.0943400</td><td>-0.073297</td><td>-0.064699</td><td>-0.26044</td></tr>\n",
              "\t<tr><td>and</td><td>0.268180</td><td> 0.143460</td><td>-0.27877</td><td> 0.016257</td><td>0.11384</td><td> 0.699230</td><td>-0.51332</td><td>-0.47368</td><td>-0.33075000</td><td>⋯</td><td>-6.9043e-02</td><td> 0.368850</td><td> 0.251680</td><td>-0.245170</td><td> 0.25381</td><td>0.136700</td><td>-0.3117800</td><td>-0.632100</td><td>-0.250280</td><td>-0.38097</td></tr>\n",
              "</tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XhSmo7FLylhL",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "6826fcb0-3aa7-4258-e6d7-f753f4acb07e"
      },
      "source": [
        "# Convert the ngrams df to vectors \n",
        "# define parameters of Keras model\n",
        "library(keras)\n",
        "max_words = 1e4\n",
        "maxlen = 2\n",
        "dim_size = 50\n",
        "# tokenize the input data and then fit the created object\n",
        "X_seg = text_tokenizer(num_words = max_words ) %>%\n",
        "  fit_text_tokenizer( paste( c(df$X1, df$X2 , df$y ) , sep=\" \") )\n",
        "X_seg"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<keras_preprocessing.text.Tokenizer>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "R-yzuDhsEsg2",
        "outputId": "3dcf2184-d1b8-41a6-9264-d5e1b367a2e3"
      },
      "source": [
        "# apply tokenizer to the text and get indices instead of words\n",
        "# later pad the sequence\n",
        "Xtrain = texts_to_sequences(X_seg , paste( df$X1, df$X2  , sep=\" \") )  %>%\n",
        " pad_sequences( maxlen = maxlen)\n",
        "tail(Xtrain,3)"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "          [,1] [,2]\n",
              "[259684,]  195  18 \n",
              "[259685,] 2380 894 \n",
              "[259686,] 2730 349 "
            ],
            "text/latex": "A matrix: 3 × 2 of type int\n\\begin{tabular}{r|ll}\n\t{[}259684,{]} &  195 &  18\\\\\n\t{[}259685,{]} & 2380 & 894\\\\\n\t{[}259686,{]} & 2730 & 349\\\\\n\\end{tabular}\n",
            "text/markdown": "\nA matrix: 3 × 2 of type int\n\n| [259684,] |  195 |  18 |\n| [259685,] | 2380 | 894 |\n| [259686,] | 2730 | 349 |\n\n",
            "text/html": [
              "<table class=\"dataframe\">\n",
              "<caption>A matrix: 3 × 2 of type int</caption>\n",
              "<tbody>\n",
              "\t<tr><th scope=row>[259684,]</th><td> 195</td><td> 18</td></tr>\n",
              "\t<tr><th scope=row>[259685,]</th><td>2380</td><td>894</td></tr>\n",
              "\t<tr><th scope=row>[259686,]</th><td>2730</td><td>349</td></tr>\n",
              "</tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "Rv9cHuTrEvv7",
        "outputId": "e415e971-835b-4538-9f2e-07927b433425"
      },
      "source": [
        "#tokenize Ytrain as well\n",
        "# tokenize the input data and then fit the created object\n",
        "# Y_seg = text_tokenizer(num_words = max_words ) %>%\n",
        "#   fit_text_tokenizer( df$y )\n",
        "# Y_seg\n",
        "# apply tokenizer to the text and get indices instead of words\n",
        "# later pad the sequence\n",
        "Ytrain = texts_to_sequences(X_seg , df$y ) %>%\n",
        " pad_sequences( maxlen = 1)\n",
        " head(Ytrain,3)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "     [,1]\n",
              "[1,] 24  \n",
              "[2,]  4  \n",
              "[3,]  1  "
            ],
            "text/latex": "A matrix: 3 × 1 of type int\n\\begin{tabular}{l}\n\t 24\\\\\n\t  4\\\\\n\t  1\\\\\n\\end{tabular}\n",
            "text/markdown": "\nA matrix: 3 × 1 of type int\n\n| 24 |\n|  4 |\n|  1 |\n\n",
            "text/html": [
              "<table class=\"dataframe\">\n",
              "<caption>A matrix: 3 × 1 of type int</caption>\n",
              "<tbody>\n",
              "\t<tr><td>24</td></tr>\n",
              "\t<tr><td> 4</td></tr>\n",
              "\t<tr><td> 1</td></tr>\n",
              "</tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "id": "_w-IMZF37TYO",
        "outputId": "e9fd0ba7-0d85-4fc0-822a-8b564ef434f5"
      },
      "source": [
        "# Unlist and make dataframe  and join with \n",
        "# downloaded pretrained word Embeddings\n",
        "# unlist word indices\n",
        "word_indices = unlist( X_seg$word_index)\n",
        "# word_indices_Y = unlist(Y_seg$word_index)\n",
        "tail(word_indices,3)"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  4dka    pip pjg1bs \n",
              " 24651  24652  24653 "
            ],
            "text/latex": "\\begin{description*}\n\\item[4dka] 24651\n\\item[pip] 24652\n\\item[pjg1bs] 24653\n\\end{description*}\n",
            "text/markdown": "4dka\n:   24651pip\n:   24652pjg1bs\n:   24653\n\n",
            "text/html": [
              "<style>\n",
              ".dl-inline {width: auto; margin:0; padding: 0}\n",
              ".dl-inline>dt, .dl-inline>dd {float: none; width: auto; display: inline-block}\n",
              ".dl-inline>dt::after {content: \":\\0020\"; padding-right: .5ex}\n",
              ".dl-inline>dt:not(:first-of-type) {padding-left: .5ex}\n",
              "</style><dl class=dl-inline><dt>4dka</dt><dd>24651</dd><dt>pip</dt><dd>24652</dd><dt>pjg1bs</dt><dd>24653</dd></dl>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 271
        },
        "id": "netaVZEpFpRV",
        "outputId": "fc1a0d8b-a82e-4a54-97ec-d0a5b341cccd"
      },
      "source": [
        "# then place them into data.frame \n",
        "dic = data.frame(word = names(word_indices) , key = word_indices, stringsAsFactors = FALSE) %>%\n",
        "  arrange(key) %>% .[1:max_words,]\n",
        "dic <- data.frame( word = unique(dic$word))\n",
        "tail(dic)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "      word   \n",
              "9995  gotchu \n",
              "9996  mammoth\n",
              "9997  flag   \n",
              "9998  43     \n",
              "9999  juiced \n",
              "10000 herc   "
            ],
            "text/latex": "A data.frame: 6 × 1\n\\begin{tabular}{r|l}\n  & word\\\\\n  & <chr>\\\\\n\\hline\n\t9995 & gotchu \\\\\n\t9996 & mammoth\\\\\n\t9997 & flag   \\\\\n\t9998 & 43     \\\\\n\t9999 & juiced \\\\\n\t10000 & herc   \\\\\n\\end{tabular}\n",
            "text/markdown": "\nA data.frame: 6 × 1\n\n| <!--/--> | word &lt;chr&gt; |\n|---|---|\n| 9995 | gotchu  |\n| 9996 | mammoth |\n| 9997 | flag    |\n| 9998 | 43      |\n| 9999 | juiced  |\n| 10000 | herc    |\n\n",
            "text/html": [
              "<table class=\"dataframe\">\n",
              "<caption>A data.frame: 6 × 1</caption>\n",
              "<thead>\n",
              "\t<tr><th></th><th scope=col>word</th></tr>\n",
              "\t<tr><th></th><th scope=col>&lt;chr&gt;</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "\t<tr><th scope=row>9995</th><td>gotchu </td></tr>\n",
              "\t<tr><th scope=row>9996</th><td>mammoth</td></tr>\n",
              "\t<tr><th scope=row>9997</th><td>flag   </td></tr>\n",
              "\t<tr><th scope=row>9998</th><td>43     </td></tr>\n",
              "\t<tr><th scope=row>9999</th><td>juiced </td></tr>\n",
              "\t<tr><th scope=row>10000</th><td>herc   </td></tr>\n",
              "</tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 295
        },
        "id": "LxCQh36IGNOh",
        "outputId": "b17e0d5a-5d1a-463c-af27-396c078656f4"
      },
      "source": [
        "# join the words with GloVe vectors and\n",
        "# if word does not exist in GloVe, then fill NA's with 0\n",
        "word_embeds = dic %>% left_join(vectors) %>% .[,2:51] %>% replace(., is.na(.), 0) %>% as.matrix()\n"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Joining, by = \"word\"\n",
            "\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "     dim_1      dim_2     dim_3     dim_4     dim_5   dim_6     dim_7   \n",
              "[1,]  0.4180000  0.249680 -0.412420  0.121700 0.34527 -0.044457 -0.49688\n",
              "[2,]  0.1189100  0.152550 -0.082073 -0.741440 0.75917 -0.483280 -0.31009\n",
              "[3,]  0.6804700 -0.039263  0.301860 -0.177920 0.42962  0.032246 -0.41376\n",
              "[4,]  0.2170500  0.465150 -0.467570  0.100820 1.01350  0.748450 -0.53104\n",
              "[5,] -0.0010919  0.333240  0.357430 -0.540410 0.82032 -0.493910 -0.32588\n",
              "[6,]  0.2681800  0.143460 -0.278770  0.016257 0.11384  0.699230 -0.51332\n",
              "     dim_8      dim_9       dim_10      ⋯ dim_41     dim_42    dim_43  \n",
              "[1,] -0.1786200 -0.00066023 -0.65660000 ⋯ -0.2987100 -0.157490 -0.34758\n",
              "[2,]  0.5147600 -0.98708000  0.00061757 ⋯ -0.0027443 -0.018298 -0.28096\n",
              "[3,]  0.1322800 -0.29847000 -0.08525300 ⋯ -0.0943750  0.018324  0.21048\n",
              "[4,] -0.2625600  0.16812000  0.13182000 ⋯  0.1381300  0.369730 -0.64289\n",
              "[5,]  0.0019972 -0.23829000  0.35554000 ⋯  0.2087700  0.115640 -0.15190\n",
              "[6,] -0.4736800 -0.33075000 -0.13834000 ⋯ -0.0690430  0.368850  0.25168\n",
              "     dim_44    dim_45    dim_46    dim_47     dim_48    dim_49    dim_50  \n",
              "[1,] -0.045637 -0.442510  0.187850  0.0027849 -0.184110 -0.115140 -0.78581\n",
              "[2,]  0.553180  0.037706  0.185550 -0.1502500 -0.575120 -0.266710  0.92121\n",
              "[3,] -0.030880 -0.197220  0.082279 -0.0943400 -0.073297 -0.064699 -0.26044\n",
              "[4,]  0.024142 -0.039315 -0.260370  0.1201700 -0.043782  0.410130  0.17960\n",
              "[5,]  0.859080  0.226200  0.165190  0.3630900 -0.456970 -0.048969  1.13160\n",
              "[6,] -0.245170  0.253810  0.136700 -0.3117800 -0.632100 -0.250280 -0.38097"
            ],
            "text/latex": "A matrix: 6 × 50 of type dbl\n\\begin{tabular}{lllllllllllllllllllll}\n dim\\_1 & dim\\_2 & dim\\_3 & dim\\_4 & dim\\_5 & dim\\_6 & dim\\_7 & dim\\_8 & dim\\_9 & dim\\_10 & ⋯ & dim\\_41 & dim\\_42 & dim\\_43 & dim\\_44 & dim\\_45 & dim\\_46 & dim\\_47 & dim\\_48 & dim\\_49 & dim\\_50\\\\\n\\hline\n\t  0.4180000 &  0.249680 & -0.412420 &  0.121700 & 0.34527 & -0.044457 & -0.49688 & -0.1786200 & -0.00066023 & -0.65660000 & ⋯ & -0.2987100 & -0.157490 & -0.34758 & -0.045637 & -0.442510 &  0.187850 &  0.0027849 & -0.184110 & -0.115140 & -0.78581\\\\\n\t  0.1189100 &  0.152550 & -0.082073 & -0.741440 & 0.75917 & -0.483280 & -0.31009 &  0.5147600 & -0.98708000 &  0.00061757 & ⋯ & -0.0027443 & -0.018298 & -0.28096 &  0.553180 &  0.037706 &  0.185550 & -0.1502500 & -0.575120 & -0.266710 &  0.92121\\\\\n\t  0.6804700 & -0.039263 &  0.301860 & -0.177920 & 0.42962 &  0.032246 & -0.41376 &  0.1322800 & -0.29847000 & -0.08525300 & ⋯ & -0.0943750 &  0.018324 &  0.21048 & -0.030880 & -0.197220 &  0.082279 & -0.0943400 & -0.073297 & -0.064699 & -0.26044\\\\\n\t  0.2170500 &  0.465150 & -0.467570 &  0.100820 & 1.01350 &  0.748450 & -0.53104 & -0.2625600 &  0.16812000 &  0.13182000 & ⋯ &  0.1381300 &  0.369730 & -0.64289 &  0.024142 & -0.039315 & -0.260370 &  0.1201700 & -0.043782 &  0.410130 &  0.17960\\\\\n\t -0.0010919 &  0.333240 &  0.357430 & -0.540410 & 0.82032 & -0.493910 & -0.32588 &  0.0019972 & -0.23829000 &  0.35554000 & ⋯ &  0.2087700 &  0.115640 & -0.15190 &  0.859080 &  0.226200 &  0.165190 &  0.3630900 & -0.456970 & -0.048969 &  1.13160\\\\\n\t  0.2681800 &  0.143460 & -0.278770 &  0.016257 & 0.11384 &  0.699230 & -0.51332 & -0.4736800 & -0.33075000 & -0.13834000 & ⋯ & -0.0690430 &  0.368850 &  0.25168 & -0.245170 &  0.253810 &  0.136700 & -0.3117800 & -0.632100 & -0.250280 & -0.38097\\\\\n\\end{tabular}\n",
            "text/markdown": "\nA matrix: 6 × 50 of type dbl\n\n| dim_1 | dim_2 | dim_3 | dim_4 | dim_5 | dim_6 | dim_7 | dim_8 | dim_9 | dim_10 | ⋯ | dim_41 | dim_42 | dim_43 | dim_44 | dim_45 | dim_46 | dim_47 | dim_48 | dim_49 | dim_50 |\n|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|\n|  0.4180000 |  0.249680 | -0.412420 |  0.121700 | 0.34527 | -0.044457 | -0.49688 | -0.1786200 | -0.00066023 | -0.65660000 | ⋯ | -0.2987100 | -0.157490 | -0.34758 | -0.045637 | -0.442510 |  0.187850 |  0.0027849 | -0.184110 | -0.115140 | -0.78581 |\n|  0.1189100 |  0.152550 | -0.082073 | -0.741440 | 0.75917 | -0.483280 | -0.31009 |  0.5147600 | -0.98708000 |  0.00061757 | ⋯ | -0.0027443 | -0.018298 | -0.28096 |  0.553180 |  0.037706 |  0.185550 | -0.1502500 | -0.575120 | -0.266710 |  0.92121 |\n|  0.6804700 | -0.039263 |  0.301860 | -0.177920 | 0.42962 |  0.032246 | -0.41376 |  0.1322800 | -0.29847000 | -0.08525300 | ⋯ | -0.0943750 |  0.018324 |  0.21048 | -0.030880 | -0.197220 |  0.082279 | -0.0943400 | -0.073297 | -0.064699 | -0.26044 |\n|  0.2170500 |  0.465150 | -0.467570 |  0.100820 | 1.01350 |  0.748450 | -0.53104 | -0.2625600 |  0.16812000 |  0.13182000 | ⋯ |  0.1381300 |  0.369730 | -0.64289 |  0.024142 | -0.039315 | -0.260370 |  0.1201700 | -0.043782 |  0.410130 |  0.17960 |\n| -0.0010919 |  0.333240 |  0.357430 | -0.540410 | 0.82032 | -0.493910 | -0.32588 |  0.0019972 | -0.23829000 |  0.35554000 | ⋯ |  0.2087700 |  0.115640 | -0.15190 |  0.859080 |  0.226200 |  0.165190 |  0.3630900 | -0.456970 | -0.048969 |  1.13160 |\n|  0.2681800 |  0.143460 | -0.278770 |  0.016257 | 0.11384 |  0.699230 | -0.51332 | -0.4736800 | -0.33075000 | -0.13834000 | ⋯ | -0.0690430 |  0.368850 |  0.25168 | -0.245170 |  0.253810 |  0.136700 | -0.3117800 | -0.632100 | -0.250280 | -0.38097 |\n\n",
            "text/html": [
              "<table class=\"dataframe\">\n",
              "<caption>A matrix: 6 × 50 of type dbl</caption>\n",
              "<thead>\n",
              "\t<tr><th scope=col>dim_1</th><th scope=col>dim_2</th><th scope=col>dim_3</th><th scope=col>dim_4</th><th scope=col>dim_5</th><th scope=col>dim_6</th><th scope=col>dim_7</th><th scope=col>dim_8</th><th scope=col>dim_9</th><th scope=col>dim_10</th><th scope=col>⋯</th><th scope=col>dim_41</th><th scope=col>dim_42</th><th scope=col>dim_43</th><th scope=col>dim_44</th><th scope=col>dim_45</th><th scope=col>dim_46</th><th scope=col>dim_47</th><th scope=col>dim_48</th><th scope=col>dim_49</th><th scope=col>dim_50</th></tr>\n",
              "</thead>\n",
              "<tbody>\n",
              "\t<tr><td> 0.4180000</td><td> 0.249680</td><td>-0.412420</td><td> 0.121700</td><td>0.34527</td><td>-0.044457</td><td>-0.49688</td><td>-0.1786200</td><td>-0.00066023</td><td>-0.65660000</td><td>⋯</td><td>-0.2987100</td><td>-0.157490</td><td>-0.34758</td><td>-0.045637</td><td>-0.442510</td><td> 0.187850</td><td> 0.0027849</td><td>-0.184110</td><td>-0.115140</td><td>-0.78581</td></tr>\n",
              "\t<tr><td> 0.1189100</td><td> 0.152550</td><td>-0.082073</td><td>-0.741440</td><td>0.75917</td><td>-0.483280</td><td>-0.31009</td><td> 0.5147600</td><td>-0.98708000</td><td> 0.00061757</td><td>⋯</td><td>-0.0027443</td><td>-0.018298</td><td>-0.28096</td><td> 0.553180</td><td> 0.037706</td><td> 0.185550</td><td>-0.1502500</td><td>-0.575120</td><td>-0.266710</td><td> 0.92121</td></tr>\n",
              "\t<tr><td> 0.6804700</td><td>-0.039263</td><td> 0.301860</td><td>-0.177920</td><td>0.42962</td><td> 0.032246</td><td>-0.41376</td><td> 0.1322800</td><td>-0.29847000</td><td>-0.08525300</td><td>⋯</td><td>-0.0943750</td><td> 0.018324</td><td> 0.21048</td><td>-0.030880</td><td>-0.197220</td><td> 0.082279</td><td>-0.0943400</td><td>-0.073297</td><td>-0.064699</td><td>-0.26044</td></tr>\n",
              "\t<tr><td> 0.2170500</td><td> 0.465150</td><td>-0.467570</td><td> 0.100820</td><td>1.01350</td><td> 0.748450</td><td>-0.53104</td><td>-0.2625600</td><td> 0.16812000</td><td> 0.13182000</td><td>⋯</td><td> 0.1381300</td><td> 0.369730</td><td>-0.64289</td><td> 0.024142</td><td>-0.039315</td><td>-0.260370</td><td> 0.1201700</td><td>-0.043782</td><td> 0.410130</td><td> 0.17960</td></tr>\n",
              "\t<tr><td>-0.0010919</td><td> 0.333240</td><td> 0.357430</td><td>-0.540410</td><td>0.82032</td><td>-0.493910</td><td>-0.32588</td><td> 0.0019972</td><td>-0.23829000</td><td> 0.35554000</td><td>⋯</td><td> 0.2087700</td><td> 0.115640</td><td>-0.15190</td><td> 0.859080</td><td> 0.226200</td><td> 0.165190</td><td> 0.3630900</td><td>-0.456970</td><td>-0.048969</td><td> 1.13160</td></tr>\n",
              "\t<tr><td> 0.2681800</td><td> 0.143460</td><td>-0.278770</td><td> 0.016257</td><td>0.11384</td><td> 0.699230</td><td>-0.51332</td><td>-0.4736800</td><td>-0.33075000</td><td>-0.13834000</td><td>⋯</td><td>-0.0690430</td><td> 0.368850</td><td> 0.25168</td><td>-0.245170</td><td> 0.253810</td><td> 0.136700</td><td>-0.3117800</td><td>-0.632100</td><td>-0.250280</td><td>-0.38097</td></tr>\n",
              "</tbody>\n",
              "</table>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ekiWJpyVnQNB",
        "outputId": "19ccd4b0-6359-4504-b38b-5da07fd64332",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "dim(word_embeds)"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "[1] 10000    50"
            ],
            "text/latex": "\\begin{enumerate*}\n\\item 10000\n\\item 50\n\\end{enumerate*}\n",
            "text/markdown": "1. 10000\n2. 50\n\n\n",
            "text/html": [
              "<style>\n",
              ".list-inline {list-style: none; margin:0; padding: 0}\n",
              ".list-inline>li {display: inline-block}\n",
              ".list-inline>li:not(:last-child)::after {content: \"\\00b7\"; padding: 0 .5ex}\n",
              "</style>\n",
              "<ol class=list-inline><li>10000</li><li>50</li></ol>\n"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqF-GF8SUA6N"
      },
      "source": [
        "# install.packages(\"sjlabelled\")\n",
        "# library(sjlabelled)\n",
        "# Ytrain <- as_factor(Ytrain)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ly7O7UJVM7M9"
      },
      "source": [
        "# Use Keras Functional API \n",
        "input = layer_input(shape = list(maxlen), name = \"input\")\n",
        "\n",
        "model = input %>%\n",
        "  layer_embedding(input_dim = max_words, output_dim = dim_size, input_length = maxlen,\n",
        "                  # put weights into list and do not allow training\n",
        "                  weights = list(word_embeds), trainable = FALSE) %>%\n",
        "  \n",
        "  layer_lstm(units = 10 , return_sequences = TRUE )%>%\n",
        "  layer_lstm(units = 5  )\n",
        "\n",
        "  lastLayer = model %>% layer_dense(units= 1  , activation='sigmoid' )\n",
        "  \n",
        "output =   lastLayer\n",
        "\n",
        "model = keras_model(input, output)\n",
        "\n",
        "# Compile and fit\n",
        "# instead of accuracy we can use \"AUC\" metrics from \"tensorflow.keras\"\n",
        "model %>% compile(\n",
        "  optimizer = \"adam\",\n",
        "  loss = \"sparse_categorical_crossentropy\"\n",
        ")"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZpMS9eH7V7OD",
        "outputId": "0ea08e33-596f-4beb-a265-a8a3bcc36089"
      },
      "source": [
        "summary(model)"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model_2\"\n",
            "________________________________________________________________________________\n",
            "Layer (type)                        Output Shape                    Param #     \n",
            "================================================================================\n",
            "input (InputLayer)                  [(None, 2)]                     0           \n",
            "________________________________________________________________________________\n",
            "embedding_2 (Embedding)             (None, 2, 50)                   500000      \n",
            "________________________________________________________________________________\n",
            "lstm_5 (LSTM)                       (None, 2, 10)                   2440        \n",
            "________________________________________________________________________________\n",
            "lstm_4 (LSTM)                       (None, 5)                       320         \n",
            "________________________________________________________________________________\n",
            "dense_2 (Dense)                     (None, 1)                       6           \n",
            "================================================================================\n",
            "Total params: 502,766\n",
            "Trainable params: 2,766\n",
            "Non-trainable params: 500,000\n",
            "________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K_jWXYYwy5W4"
      },
      "source": [
        "fit(model,\n",
        "  Xtrain,\n",
        "   Ytrain,\n",
        "  epochs = 1\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57hFaj04OEOK"
      },
      "source": [
        "evaluate(model,\n",
        "  Xtrain,\n",
        "   Ytrain)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0dT0gcgSNX8_"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}