LDA_with_W2V.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This code was originally written in google colab. \n",
    "# Check my colab in the following link:\n",
    "# https://colab.research.google.com/drive/1bLp7NvPLzTrmLfyhWPJY1iYi8sRTyvgB?usp=sharing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Process\n",
    "# 1. Read data (train and validation)\n",
    "# 2. Preprocessing & feature extraction\n",
    "# 3. BUILD AND TRAIN LDA\n",
    "# 4. BUILD AND TRAIN WordVector\n",
    "# 5. Hyperparameter tuning\n",
    "# 6. Read and process test data\n",
    "# 7. Prediction\n",
    "# 8. Export submission files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "id": "mMptQQxoE5Uc",
    "outputId": "cb06f138-b8b7-448c-fb26-a5a8d2a2b60c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: numba in /usr/local/lib/python3.6/dist-packages (0.48.0)\n",
      "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.6/dist-packages (from numba) (1.18.4)\n",
      "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from numba) (46.1.3)\n",
      "Requirement already satisfied: llvmlite<0.32.0,>=0.31.0dev0 in /usr/local/lib/python3.6/dist-packages (from numba) (0.31.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install numba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 205
    },
    "id": "MZsZCaBGGm1Z",
    "outputId": "eb3f5741-fa35-4a16-9154-f43e35395f6b"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/nltk/twitter/__init__.py:20: UserWarning: The twython library has not been installed. Some functionality from the twitter package will not be available.\n",
      "  warnings.warn(\"The twython library has not been installed. \"\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n",
      "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/wordnet.zip.\n",
      "[nltk_data] Downloading package vader_lexicon to /root/nltk_data...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importo libraries\n",
    "#  common \n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "import re\n",
    "from timeit import default_timer as timer  \n",
    "from numba import jit, cuda \n",
    "\n",
    "# languange processing imports\n",
    "import nltk\n",
    "from gensim.corpora import Dictionary\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "from nltk.stem import WordNetLemmatizer   \n",
    "from nltk.tokenize import TweetTokenizer\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "from textblob import TextBlob\n",
    "\n",
    "# LDA & W2V imports\n",
    "from gensim.models.ldamulticore import LdaMulticore\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "# hyperparameter training imports\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.metrics import classification_report\n",
    "\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "nltk.download('vader_lexicon')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 1. Read data (train and validation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 136
    },
    "id": "G2X82QchT05W",
    "outputId": "f875dbad-008a-4115-9e9b-a962ed38ab8a"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentence</th>\n",
       "      <th>label</th>\n",
       "      <th>length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>why houston flooding isn‘t a sign of climate c...</td>\n",
       "      <td>1</td>\n",
       "      <td>867</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>The U.N. Intergovernmental Panel on Climate Ch...</td>\n",
       "      <td>1</td>\n",
       "      <td>4560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Bureau Now Sets Strict Limits on CoolingOVER r...</td>\n",
       "      <td>1</td>\n",
       "      <td>2524</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            sentence  label  length\n",
       "0  why houston flooding isn‘t a sign of climate c...      1     867\n",
       "1  The U.N. Intergovernmental Panel on Climate Ch...      1    4560\n",
       "2  Bureau Now Sets Strict Limits on CoolingOVER r...      1    2524"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#train_data = pd.read_csv('../input/train.csv')\n",
    "#train_data = pd.read_csv('train_sampled.tsv', encoding=\"utf-8\", delimiter = '\\t')#, lineterminator='\\n')\n",
    "collist =  ['sentence', 'label', 'wordcount', 'web', 'email' , 'hashtag', 'sentiment', 'percent', 'subjectivity' ]\n",
    "excollist =  ['web', 'email' , 'hashtag', 'sentiment', 'percent', 'symbol' ]\n",
    "\n",
    "train_data = pd.read_csv('train_9168.tsv', encoding=\"utf-8\", delimiter = '\\t' )#, lineterminator='\\n')\n",
    "dev_data = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t' )#, lineterminator='\\n')\n",
    "\n",
    "corpus_data = pd.concat([train_data, dev_data]) \n",
    "train_data.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "2fCQj12I0f1F"
   },
   "outputs": [],
   "source": [
    "dev_data = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t' )\n",
    "train_data = pd.read_csv('train_final.tsv', encoding=\"utf-8\", delimiter = '\\t' )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 322
    },
    "id": "MaYUoFtge1-3",
    "outputId": "29b1ea57-7fff-4f04-8170-5ce591e4480a"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.84      0.62      0.71        50\n",
      "           1       0.70      0.88      0.78        50\n",
      "\n",
      "    accuracy                           0.75       100\n",
      "   macro avg       0.77      0.75      0.75       100\n",
      "weighted avg       0.77      0.75      0.75       100\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Baseline model\n",
    "baseline_model = make_pipeline(CountVectorizer(ngram_range=(1,3)), LogisticRegression()).fit(train_data['sentence'], train_data['label'])\n",
    "#baseline_model = make_pipeline(TfidfVectorizer(smooth_idf=False, sublinear_tf=False, norm=None, analyzer='word'), LogisticRegression()).fit(train_data['sentence'], train_data['label'])\n",
    "\n",
    "baseline_predicted = baseline_model.predict(dev_data['sentence'])\n",
    "print(classification_report(dev_data['label'], baseline_predicted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 2. Preprocessing & feature extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 138
    },
    "id": "Cqs8akgLUVV3",
    "outputId": "04a44413-664e-4a28-9e19-231704eda119"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:671: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "wit GPU: 24.65901859099995\n"
     ]
    }
   ],
   "source": [
    "#columnlist = ['sentence', 'label', 'wordcount', 'web', 'email' , 'sentiment', 'polarity', 'subjectivity' ]#, names= columnlist,  header=None\n",
    "\n",
    "stopwords = nltk.corpus.stopwords.words('english') \n",
    "tt = TweetTokenizer()\n",
    "sid = SentimentIntensityAnalyzer()\n",
    "start = timer() \n",
    "lemm = WordNetLemmatizer()                                      \n",
    "stemmer = PorterStemmer()\n",
    "\n",
    "for col in excollist:\n",
    "    train_data[col] = 0\n",
    "\n",
    "for idx, df in train_data.iterrows():\n",
    "    text = df['sentence']\n",
    "    train_data['web'].iloc[idx] = 1 if 'http' in text or '.net' in text or '.com' in text or 'www' in text else 0\n",
    "    train_data['email'].iloc[idx] = 1 if 'email' in text or '@' in text  else 0     \n",
    "    train_data['hashtag'].iloc[idx] = 1 if '#' in text  else 0  \n",
    "\n",
    "    text = re.sub(\"((\\S+)?(http(s)?)(\\S+))|((\\S+)?(www)(\\S+))|((\\S+)?(\\@)(\\S+)?)\", \" \", text)\n",
    "    text = re.sub(\"[^a-zA-Z ]\", \"\", text)\n",
    "    train_data['sentiment'].iloc[idx] = sid.polarity_scores(text)['compound']\n",
    "    train_data['percent'].iloc[idx] = 1 if 'percent' in text or 'per cent' in text else 0  \n",
    "    train_data['symbol'].iloc[idx] = 1 if '%' in text else 0  \n",
    "    #sentiment2 = TextBlob(text).sentiment\n",
    "    #train_data['polarity'].iloc[idx] = sentiment2.polarity\n",
    "    #train_data['subjectivity'].iloc[idx] = sentiment2.subjectivity\n",
    "\n",
    "print(\"wit GPU:\", timer()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 3. BUILD AND TRAIN LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "iuJsLlQZlLhf",
    "outputId": "b86c4477-2d48-4cef-a7c5-dd948c35ae9a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TIME : 32.60631617099989\n"
     ]
    }
   ],
   "source": [
    "def get_good_tokens(sentence):\n",
    "    replaced_punctation = list(map(lambda token: re.sub('[^0-9A-Za-z!?]+', '', token), sentence))\n",
    "    removed_punctation = list(filter(lambda token: token, replaced_punctation))\n",
    "    return removed_punctation\n",
    "\n",
    "def w2v_preprocessing(df):\n",
    "    df['sentence'] = df.sentence.str.lower()\n",
    "    df['document_sentences'] = df.sentence.str.split('.')  # split texts into individual sentences\n",
    "    df['tokenized_sentences'] = list(map(lambda sentences:\n",
    "                                         list(map(nltk.word_tokenize, sentences)),\n",
    "                                         df.document_sentences))  # tokenize sentences\n",
    "    df['tokenized_sentences'] = list(map(lambda sentences:\n",
    "                                         list(map(get_good_tokens, sentences)),\n",
    "                                         df.tokenized_sentences))  # remove unwanted characters\n",
    "    df['tokenized_sentences'] = list(map(lambda sentences:\n",
    "                                         list(filter(lambda lst: lst, sentences)),\n",
    "                                         df.tokenized_sentences))  # remove empty lists\n",
    "\n",
    "start = timer() \n",
    "w2v_preprocessing(corpus_data)\n",
    "\n",
    "def lda_get_good_tokens(df):\n",
    "    df['sentence'] = df.sentence.str.lower()\n",
    "    df['tokenized_text'] = list(map(nltk.word_tokenize, df.sentence))\n",
    "    df['tokenized_text'] = list(map(get_good_tokens, df.tokenized_text))\n",
    "\n",
    "lda_get_good_tokens(corpus_data) \n",
    "\n",
    "def remove_stopwords(df):\n",
    "    stopwords = nltk.corpus.stopwords.words('english')\n",
    "    df['stopwords_removed'] = list(map(lambda doc:\n",
    "                                       [word for word in doc if word not in stopwords],\n",
    "                                       df['tokenized_text']))\n",
    "\n",
    "remove_stopwords(corpus_data)\n",
    "\n",
    "def stem_words(df):\n",
    "    lemm = nltk.stem.WordNetLemmatizer()\n",
    "    df['lemmatized_text'] = list(map(lambda sentence:\n",
    "                                     list(map(lemm.lemmatize, sentence)),\n",
    "                                     df.stopwords_removed))\n",
    "\n",
    "    p_stemmer = nltk.stem.porter.PorterStemmer()\n",
    "    df['stemmed_text'] = list(map(lambda sentence:\n",
    "                                  list(map(p_stemmer.stem, sentence)),\n",
    "                                  df.lemmatized_text))\n",
    "\n",
    "stem_words(corpus_data)\n",
    "\n",
    "dictionary = Dictionary(documents=corpus_data.stemmed_text.values)  #train_data\n",
    "dictionary.filter_extremes(no_above=0.8, no_below=3)\n",
    "dictionary.compactify()  \n",
    "\n",
    "def document_to_bow(df):\n",
    "    df['bow'] = list(map(lambda doc: dictionary.doc2bow(doc), df.stemmed_text))\n",
    "    \n",
    "document_to_bow(corpus_data)\n",
    "\n",
    "def lda_preprocessing(df):\n",
    "    lda_get_good_tokens(df)\n",
    "    remove_stopwords(df)\n",
    "    stem_words(df)\n",
    "    document_to_bow(df)\n",
    "\n",
    "print(\"TIME :\", timer()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 104
    },
    "id": "dLennCoqUfff",
    "outputId": "951b1f7f-0511-45ef-a15d-6d97b103d324"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/gensim/models/ldamodel.py:1023: RuntimeWarning: divide by zero encountered in log\n",
      "  diff = np.log(self.expElogbeta)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 1min 7s, sys: 8.86 s, total: 1min 16s\n",
      "Wall time: 2min 29s\n"
     ]
    }
   ],
   "source": [
    "%%timeit\n",
    "corpus = corpus_data.bow  ##train_data.bow\n",
    "\n",
    "num_topics = 100\n",
    "\n",
    "def get_ldamodel(corpus, dictionary, num_topics):    \n",
    "    model = LdaMulticore(corpus=corpus,\n",
    "                          id2word=dictionary,\n",
    "                          num_topics=num_topics,\n",
    "                          workers=4,\n",
    "                          chunksize=1000, #4000\n",
    "                          passes=10,\n",
    "                          per_word_topics=True,\n",
    "                          alpha='asymmetric')\n",
    "    \n",
    "    return model\n",
    "\n",
    "LDAmodel = get_ldamodel(corpus, dictionary, num_topics)   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "cOvoNKafU4ko",
    "outputId": "6e7e2b7e-88d0-48a5-c6e7-dd7519a7af9b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time : 11.99986723300026\n"
     ]
    }
   ],
   "source": [
    "def document_to_lda_features(lda_model, document):\n",
    "    topic_importances = LDAmodel.get_document_topics(document, minimum_probability=0)\n",
    "    topic_importances = np.array(topic_importances)\n",
    "    return topic_importances[:,1]\n",
    "\n",
    "start = timer() \n",
    "train_data['lda_features'] = list(map(lambda doc:\n",
    "                                      document_to_lda_features(LDAmodel, doc),\n",
    "                                      train_data.bow))\n",
    "print(\"time :\", timer()-start) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "mNEsInmT6eBz"
   },
   "outputs": [],
   "source": [
    "## 4. BUILD AND TRAIN WordVector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "u_QRLl82U7un",
    "outputId": "1098986a-7332-4170-cc48-2dfee7eecf42"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "time : 29.260960986999635\n"
     ]
    }
   ],
   "source": [
    "sentences = []\n",
    "for sentence_group in train_data.tokenized_sentences:\n",
    "    sentences.extend(sentence_group)\n",
    "\n",
    "num_features = 200    # Word vector dimensionality\n",
    "min_word_count = 3    # Minimum word count\n",
    "num_workers = 4       # Number of threads to run in parallel\n",
    "context = 6           # Context window size\n",
    "downsampling = 1e-3   # Downsample setting for frequent words\n",
    "\n",
    "\n",
    "def getW2Vmodel(sentences, num_workers, num_features, min_word_count, context, downsampling):\n",
    "    model = Word2Vec(sentences=sentences,\n",
    "                      sg=1,\n",
    "                      hs=0,\n",
    "                      workers=num_workers,\n",
    "                      size=num_features,\n",
    "                      min_count=min_word_count,\n",
    "                      window=context,\n",
    "                      sample=downsampling,\n",
    "                      negative=5,\n",
    "                      iter=6)\n",
    "    return model\n",
    "\n",
    "start = timer() \n",
    "W2Vmodel = getW2Vmodel(sentences, num_workers, num_features, min_word_count, context, downsampling )\n",
    "print(\"time :\", timer()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 87
    },
    "id": "0hG46QXnVACL",
    "outputId": "c3cbe8b2-8902-445c-9325-ca5cd25f94f3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:9: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  if __name__ == '__main__':\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "with GPU: 15.85646318399995\n"
     ]
    }
   ],
   "source": [
    "def get_w2v_features(w2v_model, sentence_group):\n",
    "    words = np.concatenate(sentence_group)  \n",
    "    index2word_set = set(w2v_model.wv.vocab.keys())      \n",
    "    featureVec = np.zeros(w2v_model.vector_size, dtype=\"float32\")    \n",
    "\n",
    "    nwords = 0    \n",
    "    for word in words:\n",
    "        if word in index2word_set: \n",
    "            featureVec = np.add(featureVec, w2v_model[word])\n",
    "            nwords += 1.\n",
    "            \n",
    "    if nwords > 0:\n",
    "        featureVec = np.divide(featureVec, nwords)\n",
    "    return featureVec\n",
    "\n",
    "start = timer() \n",
    "train_data['w2v_features'] = list(map(lambda sen_group:\n",
    "                                      get_w2v_features(W2Vmodel, sen_group),\n",
    "                                      train_data.tokenized_sentences))\n",
    "print(\"with GPU:\", timer()-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 5. Hyperparameter tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "vdHa7hryU70S"
   },
   "outputs": [],
   "source": [
    "def get_cross_validated_model(model, param_grid, X, y, nr_folds=5):\n",
    "    #scoring='neg_log_loss'\n",
    "    grid_cv = GridSearchCV(model, param_grid=param_grid, scoring='f1', cv=nr_folds, n_jobs=-1, verbose=True)\n",
    "    best_model = grid_cv.fit(X, y) \n",
    "\n",
    "    result_df = pd.DataFrame(best_model.cv_results_)\n",
    "    show_columns = ['mean_test_score', 'mean_train_score', 'rank_test_score']\n",
    "    for col in result_df.columns:\n",
    "        if col.startswith('param_'):\n",
    "            show_columns.append(col)\n",
    "    display(result_df.sort_values(by='rank_test_score').head())\n",
    "    \n",
    "    return best_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "14ZT7VIVU723"
   },
   "outputs": [],
   "source": [
    "#X_train_metas = np.array( pd.concat([train_data.length, train_data.web, train_data.email,train_data.hashtag, train_data.sentiment, train_data.percent, train_data.symbol ], axis= 1) ) \n",
    "X_train_metas = np.array( pd.concat([ train_data.web, train_data.email,\n",
    "                                     train_data.hashtag, train_data.sentiment, train_data.percent ,train_data.symbol  ], axis= 1) ) \n",
    "X_train_lda = np.array(list(map(np.array, train_data.lda_features)))\n",
    "X_train_w2v = np.array(list(map(np.array, train_data.w2v_features)))\n",
    "X_train_ldaw2v = np.append(X_train_lda, X_train_w2v, axis=1)\n",
    "X_train_combined = np.append(X_train_metas, X_train_ldaw2v, axis=1)\n",
    "X_train_ldametas = np.append(X_train_metas, X_train_lda, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "kGRXq5bI-aYD",
    "outputId": "f8caf6a6-7fb5-4327-a877-eef478cf3c90"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    1.4s finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_penalty</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.085705</td>\n",
       "      <td>0.015394</td>\n",
       "      <td>0.002397</td>\n",
       "      <td>0.000649</td>\n",
       "      <td>l2</td>\n",
       "      <td>{'penalty': 'l2'}</td>\n",
       "      <td>0.649165</td>\n",
       "      <td>0.649412</td>\n",
       "      <td>0.736573</td>\n",
       "      <td>0.706522</td>\n",
       "      <td>0.719346</td>\n",
       "      <td>0.692203</td>\n",
       "      <td>0.036315</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.004441</td>\n",
       "      <td>0.001202</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>l1</td>\n",
       "      <td>{'penalty': 'l1'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
       "1       0.085705      0.015394  ...        0.036315                1\n",
       "0       0.004441      0.001202  ...             NaN                2\n",
       "\n",
       "[2 rows x 14 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   7 out of  10 | elapsed:    0.4s remaining:    0.2s\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    0.5s finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_penalty</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.171298</td>\n",
       "      <td>0.031425</td>\n",
       "      <td>0.003174</td>\n",
       "      <td>0.000575</td>\n",
       "      <td>l2</td>\n",
       "      <td>{'penalty': 'l2'}</td>\n",
       "      <td>0.700611</td>\n",
       "      <td>0.722793</td>\n",
       "      <td>0.803874</td>\n",
       "      <td>0.822943</td>\n",
       "      <td>0.836186</td>\n",
       "      <td>0.777281</td>\n",
       "      <td>0.054971</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.003131</td>\n",
       "      <td>0.000651</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>l1</td>\n",
       "      <td>{'penalty': 'l1'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
       "1       0.171298      0.031425  ...        0.054971                1\n",
       "0       0.003131      0.000651  ...             NaN                2\n",
       "\n",
       "[2 rows x 14 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   7 out of  10 | elapsed:    0.7s remaining:    0.3s\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    1.0s finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_penalty</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.321477</td>\n",
       "      <td>0.077043</td>\n",
       "      <td>0.003155</td>\n",
       "      <td>0.000583</td>\n",
       "      <td>l2</td>\n",
       "      <td>{'penalty': 'l2'}</td>\n",
       "      <td>0.748988</td>\n",
       "      <td>0.737705</td>\n",
       "      <td>0.847775</td>\n",
       "      <td>0.862651</td>\n",
       "      <td>0.87619</td>\n",
       "      <td>0.814662</td>\n",
       "      <td>0.059026</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.014276</td>\n",
       "      <td>0.002300</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>l1</td>\n",
       "      <td>{'penalty': 'l1'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
       "1       0.321477      0.077043  ...        0.059026                1\n",
       "0       0.014276      0.002300  ...             NaN                2\n",
       "\n",
       "[2 rows x 14 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   7 out of  10 | elapsed:    0.3s remaining:    0.1s\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    0.4s finished\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_penalty</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.119161</td>\n",
       "      <td>0.015830</td>\n",
       "      <td>0.00289</td>\n",
       "      <td>0.001141</td>\n",
       "      <td>l2</td>\n",
       "      <td>{'penalty': 'l2'}</td>\n",
       "      <td>0.697561</td>\n",
       "      <td>0.660287</td>\n",
       "      <td>0.746803</td>\n",
       "      <td>0.730667</td>\n",
       "      <td>0.686981</td>\n",
       "      <td>0.70446</td>\n",
       "      <td>0.030932</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.003204</td>\n",
       "      <td>0.000612</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>l1</td>\n",
       "      <td>{'penalty': 'l1'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
       "1       0.119161      0.015830  ...        0.030932                1\n",
       "0       0.003204      0.000612  ...             NaN                2\n",
       "\n",
       "[2 rows x 14 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 5 folds for each of 2 candidates, totalling 10 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 2 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done  10 out of  10 | elapsed:    1.1s finished\n",
      "/usr/local/lib/python3.6/dist-packages/sklearn/linear_model/_logistic.py:940: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  extra_warning_msg=_LOGISTIC_SOLVER_CONVERGENCE_MSG)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_score_time</th>\n",
       "      <th>std_score_time</th>\n",
       "      <th>param_penalty</th>\n",
       "      <th>params</th>\n",
       "      <th>split0_test_score</th>\n",
       "      <th>split1_test_score</th>\n",
       "      <th>split2_test_score</th>\n",
       "      <th>split3_test_score</th>\n",
       "      <th>split4_test_score</th>\n",
       "      <th>mean_test_score</th>\n",
       "      <th>std_test_score</th>\n",
       "      <th>rank_test_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.365882</td>\n",
       "      <td>0.078355</td>\n",
       "      <td>0.003147</td>\n",
       "      <td>0.000519</td>\n",
       "      <td>l2</td>\n",
       "      <td>{'penalty': 'l2'}</td>\n",
       "      <td>0.781893</td>\n",
       "      <td>0.743802</td>\n",
       "      <td>0.849188</td>\n",
       "      <td>0.864734</td>\n",
       "      <td>0.862651</td>\n",
       "      <td>0.820453</td>\n",
       "      <td>0.048846</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.009614</td>\n",
       "      <td>0.003052</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>l1</td>\n",
       "      <td>{'penalty': 'l1'}</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   mean_fit_time  std_fit_time  ...  std_test_score  rank_test_score\n",
       "1       0.365882      0.078355  ...        0.048846                1\n",
       "0       0.009614      0.003052  ...             NaN                2\n",
       "\n",
       "[2 rows x 14 columns]"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# store all models in a dictionary\n",
    "models = dict()\n",
    "\n",
    "# meta features only\n",
    "lr = LogisticRegression()\n",
    "param_grid = {'penalty': ['l1', 'l2']}\n",
    "models['best_lr_lda'] = get_cross_validated_model(lr, param_grid, X_train_lda, train_data.label)\n",
    "models['best_lr_w2v'] = get_cross_validated_model(lr, param_grid, X_train_w2v, train_data.label)\n",
    "models['best_lr_ldaw2v'] = get_cross_validated_model(lr, param_grid, X_train_ldaw2v, train_data.label)\n",
    "models['best_lr_ldawmetas'] = get_cross_validated_model(lr, param_grid, X_train_ldametas, train_data.label)\n",
    "models['best_lr_combined_all'] = get_cross_validated_model(lr, param_grid, X_train_combined, train_data.label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "id": "PTK0FvYjVOYG",
    "outputId": "f61ab8a2-d0bb-494c-9f45-59cb2fe04ac0"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model best_lr_lda has a test score of: -0.1626\n",
      "Model best_lr_w2v has a test score of: -0.1308\n",
      "Model best_lr_ldaw2v has a test score of: -0.1087\n",
      "Model best_lr_ldawmetas has a test score of: -0.1614\n",
      "Model best_lr_combined_all has a test score of: -0.1181\n"
     ]
    }
   ],
   "source": [
    "for name, model in models.items():\n",
    "    print(\"Model {} has a test score of: {:0.4f}\".format(name, float(model.best_score_)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 6. Read and process test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "id": "MXFlezG4437p",
    "outputId": "feca870b-9aab-4f64-d438-e8a804a9f146"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:671: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  self._setitem_with_indexer(indexer, value)\n"
     ]
    }
   ],
   "source": [
    "test_data = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t' )\n",
    "test_data = test_data.sort_values(by='label')\n",
    "test_data = test_data[:55]\n",
    "test_data = test_data.sample(frac = 1) \n",
    "\n",
    "labels = []\n",
    "\n",
    "df2 = pd.DataFrame(columns=['sentence', 'label', 'length'])\n",
    "for i, row in test_data.iterrows():\n",
    "    s = row['sentence']\n",
    "    label = row['label']\n",
    "    length = row['length']\n",
    "    labels.append(label) \n",
    "\n",
    "    df2= df2.append({'sentence': s, 'label': label, 'length': length }, ignore_index=True)\n",
    "\n",
    "\n",
    "test_data = df2\n",
    "for col in excollist:\n",
    "    test_data[col] = 0\n",
    "\n",
    "\n",
    "for idx, df in test_data.iterrows():\n",
    "    #print(idx)\n",
    "    text = df['sentence']\n",
    "    test_data['web'].iloc[idx] = 1 if 'http' in text or '.net' in text or '.com' in text or 'www' in text else 0\n",
    "    test_data['email'].iloc[idx] = 1 if 'email' in text or '@' in text  else 0     \n",
    "    test_data['hashtag'].iloc[idx] = 1 if '#' in text  else 0  \n",
    "    text = re.sub(\"((\\S+)?(http(s)?)(\\S+))|((\\S+)?(www)(\\S+))|((\\S+)?(\\@)(\\S+)?)\", \" \", text)\n",
    "    text = re.sub(\"[^a-zA-Z ]\", \"\", text)\n",
    "    test_data['sentiment'].iloc[idx] = sid.polarity_scores(text)['compound']\n",
    "    test_data['percent'].iloc[idx] = 1 if 'percent' in text or 'per cent' in text else 0  \n",
    "    test_data['symbol'].iloc[idx] = 1 if '%' in text else 0  \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 70
    },
    "id": "zbQ_LuLItES6",
    "outputId": "4766de14-5ec5-4974-c73f-1bd254ee5373"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:9: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "lda_preprocessing(test_data)\n",
    "w2v_preprocessing(test_data)\n",
    "\n",
    "test_data['lda_features'] = list(map(lambda doc:\n",
    "                                     document_to_lda_features(LDAmodel, doc),\n",
    "                                     test_data.bow))\n",
    "\n",
    "test_data['w2v_features'] = list(map(lambda sen_group:\n",
    "                                     get_w2v_features(W2Vmodel, sen_group),\n",
    "                                     test_data.tokenized_sentences))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "1hPogC1VZYPy"
   },
   "outputs": [],
   "source": [
    "X_test_metas = np.array( pd.concat([test_data.web, test_data.email,\n",
    "                                     test_data.hashtag, test_data.sentiment,test_data.percent,test_data.symbol ], axis= 1) ) \n",
    "\n",
    "X_test_lda = np.array(list(map(np.array, test_data.lda_features)))\n",
    "X_test_w2v = np.array(list(map(np.array, test_data.w2v_features)))\n",
    "X_test_ldaw2v = np.append(X_test_lda, X_test_w2v, axis=1)\n",
    "X_test_combined = np.append(X_test_metas, X_test_ldaw2v, axis=1)\n",
    "X_test_ldametas = np.append(X_test_metas, X_test_lda, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 7. Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "GMtvgp5HtqyZ"
   },
   "outputs": [],
   "source": [
    "#submission_predictions = models['best_lr_ldawmetas'].predict(X_test_ldametas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pSjnsikrZWoL"
   },
   "outputs": [],
   "source": [
    "#submission_predictions = models['best_lr_lda'].predict(X_test_lda)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "iBp5SOMn4DFf"
   },
   "outputs": [],
   "source": [
    "#submission_predictions = models['best_lr_w2v'].predict(X_test_w2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "aW1WF5-f7EvE"
   },
   "outputs": [],
   "source": [
    "#submission_predictions = models['best_lr_ldaw2v'].predict(X_test_ldaw2v)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "pglkOGvEsxnv"
   },
   "outputs": [],
   "source": [
    "submission_predictions = models['best_lr_combined_all'].predict(X_test_combined)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 50
    },
    "id": "AwzYqn5AsxqX",
    "outputId": "701141b0-11fe-4b7b-f92a-8a1b51bce091"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 0 0 0 0 0 0 0 0 0 0 0 1 0 0 1 0 0 0 0 1 0 0 0 0 1 0 0 0 0 0 0 1 1 0\n",
      " 0 0 0 1 1 1 0 0 0 0 0 0 0 1 1 0 0 1]\n"
     ]
    }
   ],
   "source": [
    "print(submission_predictions)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "5yUgjX6bUraY",
    "outputId": "ebf4c719-96e4-47ed-9dd4-c65f4e059965"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "55 55\n"
     ]
    }
   ],
   "source": [
    "test_label = test_data['label']\n",
    "print(len(test_label), len(submission_predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "id": "en_TWOVuvmd9",
    "outputId": "b9c6fa07-19e3-4a05-ad52-700b42e060fa"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " clasification report:\n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.80      0.87        50\n",
      "           1       0.23      0.60      0.33         5\n",
      "\n",
      "    accuracy                           0.78        55\n",
      "   macro avg       0.59      0.70      0.60        55\n",
      "weighted avg       0.89      0.78      0.82        55\n",
      "\n"
     ]
    }
   ],
   "source": [
    "result = classification_report(labels, submission_predictions)\n",
    "print ('\\n clasification report:\\n', result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## 8. Export submission files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 252
    },
    "id": "HnyEuNlHxRDA",
    "outputId": "4b114efe-b8e6-4740-975f-3793ffab4a8a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      prediction\n",
      "0              0\n",
      "1              0\n",
      "2              0\n",
      "3              0\n",
      "4              1\n",
      "...          ...\n",
      "1405           0\n",
      "1406           0\n",
      "1407           1\n",
      "1408           1\n",
      "1409           0\n",
      "\n",
      "[1410 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "result = np.array(submission_predictions)\n",
    "submission_df = pd.DataFrame(data=result, columns=['prediction'])\n",
    "print(submission_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "A3-FnB6P0eKv"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from google.colab import files\n",
    "\n",
    "#{\"test-0\": {\"label\": 0}, \"test-1\": {\"label\": 0}, ... ,}\n",
    "data = {}\n",
    "df = submission_df['prediction']\n",
    "for idx, rows in enumerate(df):\n",
    "    label = {}\n",
    "    _id = 'test-'+str(idx)\n",
    "    label[\"label\"] = rows\n",
    "    data[_id] = label\n",
    "\n",
    "jsonfilename = 'test-output.json'\n",
    "with open(jsonfilename, 'w') as jsonFile:\n",
    "    jsonFile.write(json.dumps(data))\n",
    "\n",
    "files.download('test-output.json')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "id": "W3grnJnw6ruJ",
    "outputId": "1b385982-8039-41ad-e463-b8b4db146fde"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1285 125\n"
     ]
    }
   ],
   "source": [
    "sdf = submission_df.sort_values(by='prediction')\n",
    "\n",
    "c0 = 0\n",
    "c1 = 0\n",
    "for i,df in sdf.iterrows():\n",
    "    result = df['prediction']\n",
    "    if result ==0:\n",
    "        c0+=1\n",
    "    else:\n",
    "        c1+=1\n",
    "\n",
    "print(c0, c1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 958
    },
    "id": "F4LGLW3vQglI",
    "outputId": "b987d8ca-76f7-4fab-e844-276ac33c5740"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                             sentence  label  length\n",
      "33  Gadget Gift Guide: Picks for Pet LoversWhile P...      0    5942\n",
      "68  With every flood, public anger over the climat...      0    5806\n",
      "34  Mass melting of Antarctic ice sheet led to thr...      0    3190\n",
      "80  EU urged to adopt meat tax to tackle climate e...      0    4729\n",
      "36  The Coalition wants to turn scientists into la...      0    5529\n",
      "78  This winter in Europe was hottest on record by...      0    2780\n",
      "41  Victorian police officer charged with murderA ...      0     807\n",
      "74  Facebook, Groupon, Netflix Drive the Next Big ...      0   13511\n",
      "45  A Top Gun fantasy that came trueIt was the sum...      0    5400\n",
      "46  Uruguay 2 Holland 3: match reportSo now Gio va...      0    5326\n",
      "47  Obama 'damaging the presidency' with immigrati...      0    4940\n",
      "48  Orange and T-Mobile merge networksThe roaming ...      0    1052\n",
      "98  Why don’t we treat the climate crisis with the...      0    6334\n",
      "30  Microsoft Wants Its Wearable On Your Wrist And...      0    3630\n",
      "50  In Chattanooga, a Young Man in a Downward Spir...      0    8185\n",
      "52  Plan to drain Congo peat bog for oil could rel...      0    6421\n",
      "53  Blaze destroys 2 Fire Island resorts and 2 nea...      0    1347\n",
      "54  G20 sounds alarm over climate emergency despit...      0    3058\n",
      "55  Toronto joins baseball's big leaguesPlay ball!...      0    4414\n",
      "56  COLLEGE FOOTBALL- Instant Analysis: College fo...      0   15699\n",
      "57  Eco-campaigners hail Heathrow as a victory, bu...      0    5979\n",
      "58  Real Estate - Homes for Sale123 real estate li...      0    1139\n",
      "59  Hazard reduction burning had little to no effe...      0    6222\n",
      "71  Ecosystems the size of Amazon 'can collapse wi...      0    5239\n",
      "70  Scientists warn of 'critical gaps' in Australi...      0    4327\n",
      "63  The government must abandon its fossil fuel po...      0    6724\n",
      "69  Collection of Vladimir Putin's most notable sp...      0    4581\n",
      "51  ART - ZORACH AND TRAJAN, 2 SCULPTORS WHO PAINT...      0    6511\n",
      "83  Stoned 'Tarzan' arrested at zooPublished: Wedn...      0    1462\n",
      "67  Tories ‘must spend £33bn more a year’ to reach...      0    3120\n",
      "27  Greenland's melting ice raised global sea leve...      0    3296\n",
      "96  Climate campaigners condemn 'insidious' cockta...      0    2991\n",
      "3   Morrison a ‘predatory’ centrist on climate pol...      0    4124\n",
      "95  Texas Man Guilty of Kidnapping Still-Missing W...      0    2820\n",
      "93  Report: Colin Kaepernick has real interest in ...      0    1206\n",
      "92  Scott Morrison's duty is to protect the Austra...      0    6276\n",
      "8   Bushfire crisis conditions eight times more li...      0    5349\n",
      "87  Seven-year-old left with one eyelash after che...      0    2863\n",
      "14  UK takes first small steps to tackle carbon fr...      0    3543\n",
      "15  Wind and solar plants will soon be cheaper tha...      0    4880\n",
      "28  Brad Pitt fights back against Angelina Jolie a...      0   26769\n",
      "18  Our government is choosing to fail on climate ...      0    4988\n",
      "19  Warm winter puts paid to German ice wine produ...      0    1800\n",
      "16  Upcoming talks in Washington, D.C. and Baltimo...      0     748\n",
      "21  The College Football Grid of AngstAnyone who f...      0    8043\n",
      "22  Trapped Americans Flown Out of Violence-Torn C...      0    4976\n",
      "25  Katie Holmes nails holiday beauty in Bobbi Bro...      0    1228\n",
      "24  Darrell G. Kirch: How to Fix the Doctor Shorta...      0    4883\n",
      "20  Inside the Liberal state stepping into a low-e...      0    5300\n",
      "85  Our House Is on Fire by Greta Thunberg et al r...      0    8215\n",
      "73  Confusion at Science?A recent Science article ...      1    4402\n",
      "97  Forget the climate emergency: they need a Pied...      1    2556\n",
      "82  A Major Threat to Our Economy – Environmental ...      1    5181\n",
      "81  Winston Steps into Climate RowDarling Jacinda....      1    3995\n",
      "84  The NZ Climate Change Curriculum is Cult Indoc...      1   18223\n"
     ]
    }
   ],
   "source": [
    "test_data = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t' )\n",
    "test_data = test_data.sort_values(by='label')\n",
    "test_data = test_data[:55]\n",
    "print(test_data)\n",
    "#test_data = test_data.sample(frac = 1) "
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "LDA+W2V.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}