LLDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "id": "0Nxw1hIdpYf-",
    "outputId": "16715d5c-ef82-4d9d-ca9b-0bbbc3e8375c"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
      "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
      "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
      "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 1,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "import gensim\n",
    "from gensim.models import LdaModel\n",
    "from gensim import models, corpora, similarities\n",
    "import re\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "import time\n",
    "import nltk\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Preprocess"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "oH7JY4s-pZoA"
   },
   "outputs": [],
   "source": [
    "def initial_clean(text):\n",
    "    text = re.sub(\"((\\S+)?(http(s)?)(\\S+))|((\\S+)?(www)(\\S+))|((\\S+)?(\\@)(\\S+)?)\", \" \", text)\n",
    "    text = re.sub(\"[^a-zA-Z ]\", \"\", text)\n",
    "    text = text.lower() # lower case the text\n",
    "    text = nltk.word_tokenize(text)\n",
    "    \n",
    "    return text\n",
    "\n",
    "stop_words = stopwords.words('english')\n",
    "def remove_stop_words(text):\n",
    "    return [word for word in text if word not in stop_words]\n",
    "\n",
    "stemmer = PorterStemmer()\n",
    "def stem_words(text):\n",
    "    try:\n",
    "        text = [stemmer.stem(word) for word in text]\n",
    "        text = [word for word in text if len(word) > 1] # make sure we have no 1 letter words\n",
    "    except IndexError: # the word \"oed\" broke this, so needed try except\n",
    "        pass\n",
    "    return text\n",
    "\n",
    "def apply_all(text):\n",
    "    return stem_words(remove_stop_words(initial_clean(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read and process train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "id": "4z9jiEs5pZqn",
    "outputId": "15c72c76-2d14-414a-f676-d829bbdd973b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2418\n",
      "2418\n",
      "Time to clean and tokenize 2418 articles: 0.5670200069745381 min\n"
     ]
    }
   ],
   "source": [
    "train_b = pd.read_csv('train_final.tsv', encoding=\"utf-8\", delimiter = '\\t')#, lineterminator='\\n')\n",
    "dev_b = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t')#, lineterminator='\\n')\n",
    "\n",
    "train_b = pd.concat([train_b, dev_b ])\n",
    "(x_train, y_train) = (train_b['sentence'], train_b['label'])\n",
    "\n",
    "#test_b = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t' )\n",
    "#(x_test, y_test) = (test_b['sentence'], test_b['label'])\n",
    "\n",
    "print(len(x_train))\n",
    "print(len(y_train))\n",
    "\n",
    "# clean text and title and create new column \"tokenized\"\n",
    "t1 = time.time()\n",
    "x_train= x_train.apply(apply_all)\n",
    "#x_test = x_test.apply(apply_all)\n",
    "t2 = time.time()\n",
    "print(\"Time to clean and tokenize\", len(x_train), \"articles:\", (t2-t1)/60, \"min\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## LLDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "SFwyGuqcpZtv"
   },
   "outputs": [],
   "source": [
    "from optparse import OptionParser\n",
    "import sys, re, numpy\n",
    "\n",
    "def load_corpus(filename):\n",
    "    corpus = []\n",
    "    labels = []\n",
    "    labelmap = dict()\n",
    "    f = open(filename, 'r') \n",
    "    for line in f:\n",
    "        mt = re.match(r'\\[(.+?)\\](.+)', line)\n",
    "        if mt:\n",
    "            label = mt.group(1).split(',')\n",
    "            for x in label: labelmap[x] = 1\n",
    "            line = mt.group(2)\n",
    "        else:\n",
    "            label = None\n",
    "        doc = re.findall(r'\\w+(?:\\'\\w+)?',line.lower())\n",
    "        if len(doc)>0:\n",
    "            corpus.append(doc)\n",
    "            labels.append(label)\n",
    "    f.close()\n",
    "    return labelmap.keys(), corpus, labels\n",
    "\n",
    "class LLDA:\n",
    "    def __init__(self, K, alpha, beta):\n",
    "        #self.K = K\n",
    "        self.alpha = alpha\n",
    "        self.beta = beta\n",
    "\n",
    "    def term_to_id(self, term):\n",
    "        if term not in self.vocas_id:\n",
    "            voca_id = len(self.vocas)\n",
    "            self.vocas_id[term] = voca_id\n",
    "            self.vocas.append(term)\n",
    "        else:\n",
    "            voca_id = self.vocas_id[term]\n",
    "        return voca_id\n",
    "\n",
    "    def complement_label(self, label):\n",
    "        if not label: return numpy.ones(len(self.labelmap))\n",
    "        vec = numpy.zeros(len(self.labelmap))\n",
    "        vec[0] = 1.0\n",
    "        for x in label: vec[self.labelmap[x]] = 1.0\n",
    "        return vec\n",
    "\n",
    "    def set_corpus(self, labelset, corpus, labels):\n",
    "        labelset.insert(0, \"common\")\n",
    "        self.labelmap = dict(zip(labelset, range(len(labelset))))\n",
    "        self.K = len(self.labelmap)\n",
    "\n",
    "        self.vocas = []\n",
    "        self.vocas_id = dict()\n",
    "        self.labels = numpy.array([self.complement_label(label) for label in labels])\n",
    "        self.docs = [[self.term_to_id(term) for term in doc] for doc in corpus]\n",
    "\n",
    "        M = len(corpus)\n",
    "        V = len(self.vocas)\n",
    "\n",
    "        self.z_m_n = []\n",
    "        self.n_m_z = numpy.zeros((M, self.K), dtype=int)\n",
    "        self.n_z_t = numpy.zeros((self.K, V), dtype=int)\n",
    "        self.n_z = numpy.zeros(self.K, dtype=int)\n",
    "\n",
    "        for m, doc, label in zip(range(M), self.docs, self.labels):\n",
    "            N_m = len(doc)\n",
    "            #z_n = [label[x] for x in numpy.random.randint(len(label), size=N_m)]\n",
    "            z_n = [numpy.random.multinomial(1, label / label.sum()).argmax() for x in range(N_m)]\n",
    "            self.z_m_n.append(z_n)\n",
    "            for t, z in zip(doc, z_n):\n",
    "                self.n_m_z[m, z] += 1\n",
    "                self.n_z_t[z, t] += 1\n",
    "                self.n_z[z] += 1\n",
    "\n",
    "    def inference(self):\n",
    "        V = len(self.vocas)\n",
    "        for m, doc, label in zip(range(len(self.docs)), self.docs, self.labels):\n",
    "            for n in range(len(doc)):\n",
    "                t = doc[n]\n",
    "                z = self.z_m_n[m][n]\n",
    "                self.n_m_z[m, z] -= 1\n",
    "                self.n_z_t[z, t] -= 1\n",
    "                self.n_z[z] -= 1\n",
    "\n",
    "                denom_a = self.n_m_z[m].sum() + self.K * self.alpha\n",
    "                denom_b = self.n_z_t.sum(axis=1) + V * self.beta\n",
    "                p_z = label * (self.n_z_t[:, t] + self.beta) / denom_b * (self.n_m_z[m] + self.alpha) / denom_a\n",
    "                new_z = numpy.random.multinomial(1, p_z / p_z.sum()).argmax()\n",
    "\n",
    "                self.z_m_n[m][n] = new_z\n",
    "                self.n_m_z[m, new_z] += 1\n",
    "                self.n_z_t[new_z, t] += 1\n",
    "                self.n_z[new_z] += 1\n",
    "\n",
    "    def phi(self):\n",
    "        V = len(self.vocas)\n",
    "        return (self.n_z_t + self.beta) / (self.n_z[:, numpy.newaxis] + V * self.beta)\n",
    "\n",
    "    def theta(self):\n",
    "        \"\"\"document-topic distribution\"\"\"\n",
    "        n_alpha = self.n_m_z + self.labels * self.alpha\n",
    "        return n_alpha / n_alpha.sum(axis=1)[:, numpy.newaxis]\n",
    "\n",
    "    def perplexity(self, docs=None):\n",
    "        if docs == None: docs = self.docs\n",
    "        phi = self.phi()\n",
    "        thetas = self.theta()\n",
    "\n",
    "        log_per = N = 0\n",
    "        for doc, theta in zip(docs, thetas):\n",
    "            for w in doc:\n",
    "                log_per -= numpy.log(numpy.inner(phi[:,w], theta))\n",
    "            N += len(doc)\n",
    "        return numpy.exp(log_per / N)\n",
    "\n",
    "def main():\n",
    "    parser = OptionParser()\n",
    "    parser.add_option(\"-f\", dest=\"filename\", help=\"corpus filename\")\n",
    "    parser.add_option(\"--alpha\", dest=\"alpha\", type=\"float\", help=\"parameter alpha\", default=0.001)\n",
    "    parser.add_option(\"--beta\", dest=\"beta\", type=\"float\", help=\"parameter beta\", default=0.001)\n",
    "    parser.add_option(\"-k\", dest=\"K\", type=\"int\", help=\"number of topics\", default=20)\n",
    "    parser.add_option(\"-i\", dest=\"iteration\", type=\"int\", help=\"iteration count\", default=100)\n",
    "    (options, args) = parser.parse_args()\n",
    "    if not options.filename: parser.error(\"need corpus filename(-f)\")\n",
    "\n",
    "    labelset, corpus, labels = load_corpus(options.filename)\n",
    "\n",
    "    llda = LLDA(options.K, options.alpha, options.beta)\n",
    "    llda.set_corpus(labelset, corpus, labels)\n",
    "\n",
    "    for i in range(options.iteration):\n",
    "        sys.stderr.write(\"-- %d \" % (i + 1))\n",
    "        llda.inference()\n",
    "    #print llda.z_m_n\n",
    "\n",
    "    phi = llda.phi()\n",
    "    for v, voca in enumerate(llda.vocas):\n",
    "        #print ','.join([voca]+[str(x) for x in llda.n_z_t[:,v]])\n",
    "        print (','.join([voca]+[str(x) for x in phi[:,v]]) )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read and process validation data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "7ssWyrvsr12G"
   },
   "outputs": [],
   "source": [
    "test_data = pd.read_csv('dev.tsv', encoding=\"utf-8\", delimiter = '\\t' )\n",
    "#(x_test, y_test) = (test_b['sentence'], test_b['label'])\n",
    "'''\n",
    "testSet = np.array(x_test)\n",
    "testLabel = np.array(y_test)\n",
    "testCorpus = list(testSet)\n",
    "testSet = np.array(x_test)\n",
    "testLabel = np.array(y_test)\n",
    "'''\n",
    "\n",
    "test_data = test_data.sort_values(by='label')\n",
    "test_data = test_data[:55]\n",
    "\n",
    "test_data = test_data.sample(frac = 1) \n",
    "labels = []\n",
    "\n",
    "df2 = pd.DataFrame(columns=['sentence', 'label'])\n",
    "for i, row in test_data.iterrows():\n",
    "    s = row['sentence']\n",
    "    label = row['label']\n",
    "    #length = row['length']\n",
    "    labels.append(label) \n",
    "    df2= df2.append({'sentence': s, 'label': label}, ignore_index=True)\n",
    "\n",
    "(x_test, y_test) = (df2['sentence'], df2['label'])\n",
    "x_test = x_test.apply(apply_all)\n",
    "testSet = np.array(x_test)\n",
    "testLabel = np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Read test data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "RSoLM3TDxpeE"
   },
   "outputs": [],
   "source": [
    "test_b = pd.read_csv('test.tsv', encoding=\"utf-8\", delimiter = '\\t' )\n",
    "\n",
    "#(x_train, y_train) = (train_b['sentence'], train_b['label'])\n",
    "(x_test, y_test) = (test_b['sentence'], test_b['label'])\n",
    "x_test = x_test.apply(apply_all)\n",
    "testSet = np.array(x_test)\n",
    "testLabel = np.array(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "U2rnZL__pZwn",
    "outputId": "b85503a6-94fe-4686-85c1-ec2f9d5eb043"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "-- 0 : 3386.6958\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- label 0 : common\n",
      "climat: 0.0083\n",
      "chang: 0.0060\n",
      "trump: 0.0058\n",
      "said: 0.0053\n",
      "year: 0.0051\n",
      "would: 0.0051\n",
      "one: 0.0044\n",
      "like: 0.0041\n",
      "state: 0.0039\n",
      "peopl: 0.0039\n",
      "new: 0.0038\n",
      "say: 0.0037\n",
      "time: 0.0034\n",
      "also: 0.0033\n",
      "presid: 0.0032\n",
      "us: 0.0030\n",
      "nation: 0.0028\n",
      "use: 0.0028\n",
      "even: 0.0028\n",
      "world: 0.0027\n",
      "\n",
      "-- label 1 : 0\n",
      "trump: 0.0111\n",
      "said: 0.0084\n",
      "say: 0.0053\n",
      "one: 0.0049\n",
      "would: 0.0048\n",
      "peopl: 0.0047\n",
      "like: 0.0043\n",
      "state: 0.0040\n",
      "presid: 0.0038\n",
      "year: 0.0036\n",
      "new: 0.0033\n",
      "time: 0.0031\n",
      "clinton: 0.0031\n",
      "also: 0.0030\n",
      "polit: 0.0028\n",
      "make: 0.0027\n",
      "could: 0.0027\n",
      "climat: 0.0027\n",
      "obama: 0.0026\n",
      "go: 0.0026\n",
      "\n",
      "-- label 2 : 1\n",
      "climat: 0.0264\n",
      "chang: 0.0116\n",
      "warm: 0.0084\n",
      "global: 0.0074\n",
      "temperatur: 0.0072\n",
      "energi: 0.0070\n",
      "carbon: 0.0059\n",
      "emiss: 0.0053\n",
      "year: 0.0047\n",
      "increas: 0.0044\n",
      "would: 0.0042\n",
      "scienc: 0.0041\n",
      "new: 0.0037\n",
      "polici: 0.0036\n",
      "govern: 0.0036\n",
      "us: 0.0036\n",
      "dioxid: 0.0036\n",
      "report: 0.0036\n",
      "per: 0.0034\n",
      "human: 0.0034\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "-- 1 : 3290.8821\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- label 0 : common\n",
      "climat: 0.0078\n",
      "chang: 0.0058\n",
      "would: 0.0054\n",
      "said: 0.0053\n",
      "year: 0.0050\n",
      "trump: 0.0048\n",
      "one: 0.0047\n",
      "like: 0.0043\n",
      "state: 0.0043\n",
      "new: 0.0041\n",
      "peopl: 0.0039\n",
      "say: 0.0037\n",
      "time: 0.0036\n",
      "also: 0.0033\n",
      "us: 0.0032\n",
      "nation: 0.0031\n",
      "presid: 0.0031\n",
      "world: 0.0029\n",
      "even: 0.0028\n",
      "use: 0.0028\n",
      "\n",
      "-- label 1 : 0\n",
      "trump: 0.0127\n",
      "said: 0.0088\n",
      "say: 0.0055\n",
      "peopl: 0.0048\n",
      "one: 0.0047\n",
      "would: 0.0045\n",
      "like: 0.0043\n",
      "presid: 0.0041\n",
      "clinton: 0.0037\n",
      "state: 0.0036\n",
      "year: 0.0035\n",
      "also: 0.0031\n",
      "new: 0.0031\n",
      "time: 0.0030\n",
      "obama: 0.0029\n",
      "polit: 0.0029\n",
      "get: 0.0027\n",
      "go: 0.0027\n",
      "make: 0.0027\n",
      "could: 0.0026\n",
      "\n",
      "-- label 2 : 1\n",
      "climat: 0.0283\n",
      "chang: 0.0126\n",
      "warm: 0.0090\n",
      "temperatur: 0.0081\n",
      "global: 0.0079\n",
      "energi: 0.0073\n",
      "carbon: 0.0063\n",
      "emiss: 0.0057\n",
      "year: 0.0053\n",
      "increas: 0.0049\n",
      "scienc: 0.0045\n",
      "would: 0.0041\n",
      "report: 0.0040\n",
      "dioxid: 0.0040\n",
      "use: 0.0038\n",
      "polici: 0.0038\n",
      "human: 0.0037\n",
      "govern: 0.0037\n",
      "new: 0.0036\n",
      "model: 0.0036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "-- 2 : 3246.4620\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- label 0 : common\n",
      "climat: 0.0081\n",
      "chang: 0.0062\n",
      "would: 0.0052\n",
      "year: 0.0051\n",
      "said: 0.0051\n",
      "one: 0.0046\n",
      "state: 0.0044\n",
      "like: 0.0043\n",
      "new: 0.0043\n",
      "peopl: 0.0040\n",
      "trump: 0.0040\n",
      "say: 0.0036\n",
      "time: 0.0034\n",
      "us: 0.0033\n",
      "also: 0.0033\n",
      "world: 0.0031\n",
      "nation: 0.0030\n",
      "presid: 0.0030\n",
      "even: 0.0028\n",
      "make: 0.0028\n",
      "\n",
      "-- label 1 : 0\n",
      "trump: 0.0138\n",
      "said: 0.0091\n",
      "say: 0.0058\n",
      "one: 0.0048\n",
      "peopl: 0.0048\n",
      "would: 0.0046\n",
      "presid: 0.0044\n",
      "like: 0.0043\n",
      "clinton: 0.0041\n",
      "state: 0.0035\n",
      "time: 0.0032\n",
      "year: 0.0032\n",
      "obama: 0.0032\n",
      "also: 0.0031\n",
      "polit: 0.0030\n",
      "new: 0.0028\n",
      "republican: 0.0028\n",
      "go: 0.0028\n",
      "get: 0.0027\n",
      "hous: 0.0027\n",
      "\n",
      "-- label 2 : 1\n",
      "climat: 0.0285\n",
      "chang: 0.0128\n",
      "warm: 0.0093\n",
      "temperatur: 0.0083\n",
      "global: 0.0081\n",
      "energi: 0.0073\n",
      "carbon: 0.0066\n",
      "emiss: 0.0058\n",
      "year: 0.0054\n",
      "increas: 0.0049\n",
      "scienc: 0.0046\n",
      "would: 0.0042\n",
      "report: 0.0042\n",
      "dioxid: 0.0041\n",
      "use: 0.0039\n",
      "polici: 0.0038\n",
      "human: 0.0037\n",
      "model: 0.0037\n",
      "us: 0.0037\n",
      "per: 0.0037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "-- 3 : 3211.8618\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- label 0 : common\n",
      "climat: 0.0087\n",
      "chang: 0.0066\n",
      "would: 0.0053\n",
      "said: 0.0052\n",
      "year: 0.0051\n",
      "state: 0.0045\n",
      "one: 0.0044\n",
      "like: 0.0043\n",
      "new: 0.0043\n",
      "peopl: 0.0038\n",
      "us: 0.0036\n",
      "say: 0.0035\n",
      "time: 0.0034\n",
      "world: 0.0034\n",
      "trump: 0.0033\n",
      "also: 0.0031\n",
      "nation: 0.0030\n",
      "govern: 0.0029\n",
      "presid: 0.0028\n",
      "global: 0.0028\n",
      "\n",
      "-- label 1 : 0\n",
      "trump: 0.0147\n",
      "said: 0.0090\n",
      "say: 0.0059\n",
      "one: 0.0050\n",
      "peopl: 0.0050\n",
      "presid: 0.0046\n",
      "would: 0.0045\n",
      "like: 0.0042\n",
      "clinton: 0.0042\n",
      "state: 0.0033\n",
      "obama: 0.0032\n",
      "also: 0.0031\n",
      "polit: 0.0031\n",
      "time: 0.0031\n",
      "year: 0.0031\n",
      "republican: 0.0029\n",
      "go: 0.0029\n",
      "hous: 0.0028\n",
      "new: 0.0028\n",
      "get: 0.0027\n",
      "\n",
      "-- label 2 : 1\n",
      "climat: 0.0285\n",
      "chang: 0.0127\n",
      "warm: 0.0094\n",
      "temperatur: 0.0085\n",
      "global: 0.0081\n",
      "energi: 0.0073\n",
      "carbon: 0.0067\n",
      "emiss: 0.0059\n",
      "year: 0.0057\n",
      "increas: 0.0049\n",
      "report: 0.0046\n",
      "scienc: 0.0046\n",
      "would: 0.0044\n",
      "dioxid: 0.0041\n",
      "use: 0.0038\n",
      "polici: 0.0038\n",
      "model: 0.0038\n",
      "record: 0.0038\n",
      "human: 0.0037\n",
      "claim: 0.0037\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "-- 4 : 3185.3085\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "-- label 0 : common\n",
      "climat: 0.0092\n",
      "chang: 0.0068\n",
      "would: 0.0055\n",
      "year: 0.0052\n",
      "said: 0.0051\n",
      "state: 0.0047\n",
      "new: 0.0044\n",
      "one: 0.0043\n",
      "like: 0.0040\n",
      "peopl: 0.0038\n",
      "world: 0.0037\n",
      "us: 0.0037\n",
      "time: 0.0034\n",
      "say: 0.0032\n",
      "govern: 0.0031\n",
      "trump: 0.0030\n",
      "global: 0.0030\n",
      "also: 0.0030\n",
      "nation: 0.0029\n",
      "polici: 0.0028\n",
      "\n",
      "-- label 1 : 0\n",
      "trump: 0.0151\n",
      "said: 0.0090\n",
      "say: 0.0061\n",
      "one: 0.0051\n",
      "peopl: 0.0049\n",
      "presid: 0.0048\n",
      "like: 0.0046\n",
      "clinton: 0.0043\n",
      "would: 0.0042\n",
      "obama: 0.0033\n",
      "also: 0.0033\n",
      "time: 0.0032\n",
      "republican: 0.0031\n",
      "polit: 0.0030\n",
      "state: 0.0030\n",
      "hous: 0.0029\n",
      "campaign: 0.0029\n",
      "year: 0.0029\n",
      "work: 0.0028\n",
      "go: 0.0028\n",
      "\n",
      "-- label 2 : 1\n",
      "climat: 0.0283\n",
      "chang: 0.0127\n",
      "warm: 0.0096\n",
      "temperatur: 0.0087\n",
      "global: 0.0080\n",
      "energi: 0.0073\n",
      "carbon: 0.0068\n",
      "emiss: 0.0060\n",
      "year: 0.0058\n",
      "increas: 0.0048\n",
      "report: 0.0047\n",
      "scienc: 0.0045\n",
      "dioxid: 0.0043\n",
      "would: 0.0043\n",
      "use: 0.0039\n",
      "model: 0.0038\n",
      "claim: 0.0038\n",
      "record: 0.0038\n",
      "per: 0.0037\n",
      "polici: 0.0037\n",
      "perplexity : 3166.9508\n",
      "\n",
      "-- label 0 : common\n",
      "climat: 0.0092\n",
      "chang: 0.0068\n",
      "would: 0.0055\n",
      "year: 0.0052\n",
      "said: 0.0051\n",
      "state: 0.0047\n",
      "new: 0.0044\n",
      "one: 0.0043\n",
      "like: 0.0040\n",
      "peopl: 0.0038\n",
      "world: 0.0037\n",
      "us: 0.0037\n",
      "time: 0.0034\n",
      "say: 0.0032\n",
      "govern: 0.0031\n",
      "trump: 0.0030\n",
      "global: 0.0030\n",
      "also: 0.0030\n",
      "nation: 0.0029\n",
      "polici: 0.0028\n",
      "use: 0.0027\n",
      "energi: 0.0027\n",
      "make: 0.0027\n",
      "could: 0.0027\n",
      "presid: 0.0027\n",
      "power: 0.0026\n",
      "even: 0.0026\n",
      "scienc: 0.0025\n",
      "get: 0.0024\n",
      "go: 0.0023\n",
      "countri: 0.0023\n",
      "polit: 0.0023\n",
      "take: 0.0021\n",
      "research: 0.0020\n",
      "administr: 0.0020\n",
      "need: 0.0020\n",
      "american: 0.0020\n",
      "environment: 0.0020\n",
      "scientist: 0.0019\n",
      "way: 0.0019\n",
      "\n",
      "-- label 1 : 0\n",
      "trump: 0.0151\n",
      "said: 0.0090\n",
      "say: 0.0061\n",
      "one: 0.0051\n",
      "peopl: 0.0049\n",
      "presid: 0.0048\n",
      "like: 0.0046\n",
      "clinton: 0.0043\n",
      "would: 0.0042\n",
      "obama: 0.0033\n",
      "also: 0.0033\n",
      "time: 0.0032\n",
      "republican: 0.0031\n",
      "polit: 0.0030\n",
      "state: 0.0030\n",
      "hous: 0.0029\n",
      "campaign: 0.0029\n",
      "year: 0.0029\n",
      "work: 0.0028\n",
      "go: 0.0028\n",
      "make: 0.0027\n",
      "new: 0.0027\n",
      "get: 0.0026\n",
      "want: 0.0026\n",
      "american: 0.0026\n",
      "democrat: 0.0025\n",
      "think: 0.0025\n",
      "support: 0.0025\n",
      "first: 0.0024\n",
      "two: 0.0024\n",
      "could: 0.0024\n",
      "mani: 0.0023\n",
      "day: 0.0023\n",
      "white: 0.0023\n",
      "call: 0.0023\n",
      "right: 0.0023\n",
      "even: 0.0022\n",
      "donald: 0.0022\n",
      "report: 0.0021\n",
      "know: 0.0021\n",
      "\n",
      "-- label 2 : 1\n",
      "climat: 0.0283\n",
      "chang: 0.0127\n",
      "warm: 0.0096\n",
      "temperatur: 0.0087\n",
      "global: 0.0080\n",
      "energi: 0.0073\n",
      "carbon: 0.0068\n",
      "emiss: 0.0060\n",
      "year: 0.0058\n",
      "increas: 0.0048\n",
      "report: 0.0047\n",
      "scienc: 0.0045\n",
      "dioxid: 0.0043\n",
      "would: 0.0043\n",
      "use: 0.0039\n",
      "model: 0.0038\n",
      "claim: 0.0038\n",
      "record: 0.0038\n",
      "per: 0.0037\n",
      "polici: 0.0037\n",
      "human: 0.0036\n",
      "govern: 0.0035\n",
      "caus: 0.0035\n",
      "new: 0.0035\n",
      "data: 0.0034\n",
      "cost: 0.0034\n",
      "scientist: 0.0034\n",
      "electr: 0.0034\n",
      "show: 0.0033\n",
      "us: 0.0032\n",
      "level: 0.0031\n",
      "natur: 0.0031\n",
      "co: 0.0030\n",
      "solar: 0.0028\n",
      "australia: 0.0028\n",
      "fuel: 0.0027\n",
      "rise: 0.0027\n",
      "power: 0.0027\n",
      "scientif: 0.0027\n",
      "world: 0.0026\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\"\\nimport shelve\\nfilename = './SavedModels/llda_prepended_hostnames_5iter.dat'\\nmy_shelf = shelve.open(filename, 'n')\\n\\nmy_shelf['llda'] = globals()['llda']\\nmy_shelf['trainSet'] = globals()['trainSet']\\nmy_shelf['testSet'] = globals()['testSet']\\n\\nmy_shelf.close()\\n\""
      ]
     },
     "execution_count": 24,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Import the LLDA (labeled latent dirichlet allocation) class \n",
    "\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import matthews_corrcoef\n",
    "import sys\n",
    "\n",
    "trainSet = np.array(x_train)\n",
    "trainLabel =  np.array(y_train)\n",
    "#testSet = np.array(x_test)\n",
    "#testLabel = np.array(y_test)\n",
    "\n",
    "#corpus = map(methodcaller(\"split\", \" \"), list(trainSet[:, 0]))\n",
    "corpus = list(trainSet)\n",
    "\n",
    "# Thanks http://stackoverflow.com/questions/25346058/removing-list-of-words-from-a-string\n",
    "# Allows for removing stupid words that do nothing about distinguishing topics from every headline\n",
    "stopwords = ['the', 'a', 'at', 'of', '...', ':', ',', 'in', 'for', 'with', 'to', 'on', 'and', 'is', ';', '-', '–', '\\'', '\"', '|', \n",
    "             'are', '&', 'this', 'about', 'from', 'be', 'as', 'by', 'up', 'what', 'will', 'how', 'that', 'you', 'it', 'why', 'after']\n",
    "#trimmed_corpus = []\n",
    "#for i in range(len(corpus)):\n",
    "#    temp = [word for word in corpus[i].split() if word.lower() not in stopwords]\n",
    "#    trimmed_corpus.append(temp)\n",
    "\n",
    "#corpus = trimmed_corpus\n",
    "\n",
    "# Turn the 1.0/0.0 float labels into 1 or 0\n",
    "\n",
    "labels = map(list, map(str, map(int, trainLabel)))\n",
    "\n",
    "# '0' for real and '1' for fake\n",
    "labelset = ['0', '1']\n",
    "\n",
    "# K is the number of topics, so we want that to be 2, not the default of 20\n",
    "K = 2\n",
    "\n",
    "# Alpha and Beta: controlling similarity between topics and words within a topic\n",
    "# explained : https://www.youtube.com/watch?v=3mHy4OSyRf0\n",
    "# as in the llda_nltk example\n",
    "alpha = 0.0001\n",
    "beta = 0.0001\n",
    "iterations = 5\n",
    "\n",
    "# Instantiate an LLDA object\n",
    "llda = LLDA(K, alpha, beta)\n",
    "llda.set_corpus(labelset, corpus, labels)\n",
    "\n",
    "# lower perplexity in each step is good\n",
    "\n",
    "for i in range(iterations):\n",
    "    sys.stderr.write(\"-- %d : %.4f\\n\" % (i, llda.perplexity()))\n",
    "    llda.inference()\n",
    "    \n",
    "    phi = llda.phi()\n",
    "    for k, label in enumerate(labelset):\n",
    "        print (\"\\n-- label %d : %s\" % (k, label))\n",
    "        for w in np.argsort(-phi[k])[:20]:\n",
    "            print (\"%s: %.4f\" % (llda.vocas[w], phi[k,w]))\n",
    "print (\"perplexity : %.4f\" % llda.perplexity() )\n",
    "\n",
    "# This prints out the 40 most likely words in each topic\n",
    "phi = llda.phi()\n",
    "for k, label in enumerate(labelset):\n",
    "    print (\"\\n-- label %d : %s\" % (k, label))\n",
    "    for w in np.argsort(-phi[k])[:40]:\n",
    "        print (\"%s: %.4f\" % (llda.vocas[w], phi[k,w]))        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BcfNNAAFpZzP"
   },
   "outputs": [],
   "source": [
    "testCorpus = list(testSet)\n",
    " \n",
    "trimmedTestCorpus = testCorpus\n",
    "phi = llda.phi()\n",
    "totalPredictionsCorrect = 0\n",
    "pred_classes = np.zeros([np.size(testSet, axis=0)])\n",
    "true_classes = np.float64(np.reshape(testLabel, [np.size(testSet, axis=0)]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "S1wN0dcEpZ12",
    "outputId": "971c9757-ac52-4b5c-adb9-99464736bc8b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:  0.07166381530781253 1:  0.09726452735205175 label:  -1\n",
      "0:  0.16982709122789905 1:  0.0724875048984593 label:  -1\n",
      "0:  0.1609432065312889 1:  0.12158975400964198 label:  -1\n",
      "0:  0.08852082877751154 1:  0.0645523178031501 label:  -1\n",
      "0:  0.062013034060428254 1:  0.14268834044939072 label:  -1\n",
      "0:  0.15480556163339906 1:  0.09384829282320577 label:  -1\n",
      "0:  0.07938497915443109 1:  0.03734410665845324 label:  -1\n",
      "0:  0.09502942328516847 1:  0.06183646425303698 label:  -1\n",
      "0:  0.16004347591085114 1:  0.15143248511292642 label:  -1\n",
      "0:  0.08190703142304509 1:  0.056047777943513385 label:  -1\n",
      "0:  0.10411821998765398 1:  0.05154699038828512 label:  -1\n",
      "i = 10, 0.7092198581560284% done\n",
      "0:  0.08320542925135987 1:  0.03424868083305151 label:  -1\n",
      "0:  0.13855758341444352 1:  0.08893623708610068 label:  -1\n",
      "0:  0.1876420231718467 1:  0.1583102139604155 label:  -1\n",
      "0:  0.04171484064303213 1:  0.02707267127617511 label:  -1\n",
      "0:  0.0744156603623152 1:  0.22139379042155025 label:  -1\n",
      "0:  0.22579387510060164 1:  0.1551199015056018 label:  -1\n",
      "0:  0.04523906486230182 1:  0.031383691899017484 label:  -1\n",
      "0:  0.1436266257615012 1:  0.1222630628042344 label:  -1\n",
      "0:  0.07731142237306902 1:  0.07218922870795628 label:  -1\n",
      "0:  0.12333949336003482 1:  0.05927429976658543 label:  -1\n",
      "i = 20, 1.4184397163120568% done\n",
      "0:  0.15910218024384298 1:  0.09462102746652096 label:  -1\n",
      "0:  0.060493154296157474 1:  0.03488582364067515 label:  -1\n",
      "0:  0.1387015604457924 1:  0.10337410234927223 label:  -1\n",
      "0:  0.10645480077350969 1:  0.06959092715539568 label:  -1\n",
      "0:  0.22741062194581008 1:  0.14005390725338226 label:  -1\n",
      "0:  0.1726786158191844 1:  0.13020726959266887 label:  -1\n",
      "0:  0.18058526083809318 1:  0.18981133268388953 label:  -1\n",
      "0:  0.14377056265050367 1:  0.08697505936184953 label:  -1\n",
      "0:  0.12573695624325718 1:  0.09329697311869081 label:  -1\n",
      "0:  0.046490403402614996 1:  0.05583086381350715 label:  -1\n",
      "i = 30, 2.127659574468085% done\n",
      "0:  0.1753252815585979 1:  0.3239544994657364 label:  -1\n",
      "0:  0.09407709622451381 1:  0.10289507259307056 label:  -1\n",
      "0:  0.17226888648800254 1:  0.1260318195667617 label:  -1\n",
      "0:  0.09602052342298745 1:  0.041266501873137656 label:  -1\n",
      "0:  0.10571560742496958 1:  0.07124480538091105 label:  -1\n",
      "0:  0.03544432817738263 1:  0.025486541068331623 label:  -1\n",
      "0:  0.10571839081221886 1:  0.24282234895839025 label:  -1\n",
      "0:  0.03936720631602846 1:  0.08007466695377369 label:  -1\n",
      "0:  0.10329876547794684 1:  0.11404768454540447 label:  -1\n",
      "0:  0.07661653953075231 1:  0.03909743061578243 label:  -1\n",
      "i = 40, 2.8368794326241136% done\n",
      "0:  0.10754000702936228 1:  0.06241490404186629 label:  -1\n",
      "0:  0.07277394913027534 1:  0.043534976836682435 label:  -1\n",
      "0:  0.23043656060289297 1:  0.1549256147851077 label:  -1\n",
      "0:  0.10204742112391449 1:  0.060995965324066335 label:  -1\n",
      "0:  0.059734621472475866 1:  0.047448353262018426 label:  -1\n",
      "0:  0.17838160354243837 1:  0.0969031020124608 label:  -1\n",
      "0:  0.027875415945511954 1:  0.02119811396336894 label:  -1\n",
      "0:  0.16456707817346405 1:  0.09050884588824802 label:  -1\n",
      "0:  0.07926040278069656 1:  0.05201692950487753 label:  -1\n",
      "0:  0.19283008646873487 1:  0.15379586658897426 label:  -1\n",
      "i = 50, 3.5460992907801416% done\n",
      "0:  0.14616804325172722 1:  0.1289645547109139 label:  -1\n",
      "0:  0.06256118167692974 1:  0.039806895907686504 label:  -1\n",
      "0:  0.16622814327652138 1:  0.09308913290842184 label:  -1\n",
      "0:  0.20497522674887483 1:  0.13532719310523736 label:  -1\n",
      "0:  0.15868691043832062 1:  0.0771193816730314 label:  -1\n",
      "0:  0.1296127795234017 1:  0.08686209900266603 label:  -1\n",
      "0:  0.1537120240411749 1:  0.10170662499325024 label:  -1\n",
      "0:  0.14586075192863807 1:  0.236839370053823 label:  -1\n",
      "0:  0.16911008718425422 1:  0.13797521142321106 label:  -1\n",
      "0:  0.2049226081611564 1:  0.10722422162418481 label:  -1\n",
      "i = 60, 4.25531914893617% done\n",
      "0:  0.056146718480089355 1:  0.20934189066264913 label:  -1\n",
      "0:  0.17087082379562488 1:  0.11594114814361624 label:  -1\n",
      "0:  0.11323468501005869 1:  0.09524912030449428 label:  -1\n",
      "0:  0.13859359829714152 1:  0.07423634747819713 label:  -1\n",
      "0:  0.11774448190446392 1:  0.07462947053193004 label:  -1\n",
      "0:  0.17400746397044872 1:  0.1436012395874523 label:  -1\n",
      "0:  0.2532733969822664 1:  0.15389981810035483 label:  -1\n",
      "0:  0.13010557250521992 1:  0.07328286966261725 label:  -1\n",
      "0:  0.09420998095658713 1:  0.07923871266677503 label:  -1\n",
      "0:  0.09687044043677001 1:  0.09813215314366325 label:  -1\n",
      "i = 70, 4.964539007092198% done\n",
      "0:  0.10441167078337688 1:  0.12871597223951387 label:  -1\n",
      "0:  0.1339232419833371 1:  0.23774313607476255 label:  -1\n",
      "0:  0.1198069726867084 1:  0.14108871007460744 label:  -1\n",
      "0:  0.051044476474616345 1:  0.031135149645687973 label:  -1\n",
      "0:  0.09783939611838426 1:  0.09158881269892248 label:  -1\n",
      "0:  0.12473202360654123 1:  0.0991941338909222 label:  -1\n",
      "0:  0.1525880505218771 1:  0.10486986390825821 label:  -1\n",
      "0:  0.09666835389839677 1:  0.07053986078089201 label:  -1\n",
      "0:  0.18977925457366904 1:  0.09995333763888413 label:  -1\n",
      "0:  0.1242946129437831 1:  0.08558325835201282 label:  -1\n",
      "i = 80, 5.673758865248227% done\n",
      "0:  0.10280045638403969 1:  0.06323736494025163 label:  -1\n",
      "0:  0.06102470623610782 1:  0.04813973081560957 label:  -1\n",
      "0:  0.0933822197496038 1:  0.12403893463871153 label:  -1\n",
      "0:  0.11551311891498123 1:  0.1559106393840929 label:  -1\n",
      "0:  0.08243302655487256 1:  0.04472344070448633 label:  -1\n",
      "0:  0.20941301934751083 1:  0.15524189511456354 label:  -1\n",
      "0:  0.059612798033176075 1:  0.023299415512407516 label:  -1\n",
      "0:  0.1125176643557877 1:  0.06174160066993507 label:  -1\n",
      "0:  0.14122914100762116 1:  0.08172412254719397 label:  -1\n",
      "0:  0.09413799631672543 1:  0.061990122588285096 label:  -1\n",
      "i = 90, 6.382978723404255% done\n",
      "0:  0.16295862695178104 1:  0.1259414360055859 label:  -1\n",
      "0:  0.15482493571401162 1:  0.13069074864238323 label:  -1\n",
      "0:  0.16779785104332334 1:  0.1001160039181583 label:  -1\n",
      "0:  0.13648679479392348 1:  0.08232059494394738 label:  -1\n",
      "0:  0.07063117836148752 1:  0.04485449106142491 label:  -1\n",
      "0:  0.1605971335488786 1:  0.12614927847529703 label:  -1\n",
      "0:  0.06930233491656722 1:  0.04617401613148978 label:  -1\n",
      "0:  0.09741306059058807 1:  0.05050312268509135 label:  -1\n",
      "0:  0.17903222681778513 1:  0.3127838291478377 label:  -1\n",
      "0:  0.07697645719318538 1:  0.03938215422028743 label:  -1\n",
      "i = 100, 7.092198581560283% done\n",
      "0:  0.08194857598317662 1:  0.0623154998837883 label:  -1\n",
      "0:  0.1452738486761297 1:  0.12027023888112263 label:  -1\n",
      "0:  0.06562307541845659 1:  0.08089259495928629 label:  -1\n",
      "0:  0.19804859316747842 1:  0.17325418523092506 label:  -1\n",
      "0:  0.11267823872475032 1:  0.1475597411398323 label:  -1\n",
      "0:  0.0545880783734676 1:  0.020524811947103035 label:  -1\n",
      "0:  0.13774090896986044 1:  0.08280413499860018 label:  -1\n",
      "0:  0.15661611849567533 1:  0.07935622851325291 label:  -1\n",
      "0:  0.1591464860437223 1:  0.113261442956823 label:  -1\n",
      "0:  0.08457303636075636 1:  0.1811892408267003 label:  -1\n",
      "i = 110, 7.801418439716312% done\n",
      "0:  0.08810280686808572 1:  0.06350395652137727 label:  -1\n",
      "0:  0.04877435563437662 1:  0.11939347635728169 label:  -1\n",
      "0:  0.18030013528817893 1:  0.1416581396599685 label:  -1\n",
      "0:  0.12375197922462039 1:  0.06650901505358675 label:  -1\n",
      "0:  0.1499857293404705 1:  0.2221168534882957 label:  -1\n",
      "0:  0.2369839029949313 1:  0.15305025700074618 label:  -1\n",
      "0:  0.05913109292113313 1:  0.03841960745574305 label:  -1\n",
      "0:  0.12644843808425024 1:  0.0643489892469975 label:  -1\n",
      "0:  0.11538577659515485 1:  0.08301652572539074 label:  -1\n",
      "0:  0.1383970267634821 1:  0.07950534627352493 label:  -1\n",
      "i = 120, 8.51063829787234% done\n",
      "0:  0.14622894417447013 1:  0.12740553193450865 label:  -1\n",
      "0:  0.07046507348456002 1:  0.05285291858728552 label:  -1\n",
      "0:  0.15796159471421176 1:  0.09853439892108418 label:  -1\n",
      "0:  0.04923945621086783 1:  0.026191491994788724 label:  -1\n",
      "0:  0.10274785053113465 1:  0.07776556902764416 label:  -1\n",
      "0:  0.09474428140147191 1:  0.07729106041521591 label:  -1\n",
      "0:  0.06486175782330687 1:  0.10897746559726136 label:  -1\n",
      "0:  0.16663230998586123 1:  0.07207176076165231 label:  -1\n",
      "0:  0.11998690646356389 1:  0.1585812873710209 label:  -1\n",
      "0:  0.11961039146351708 1:  0.09259652616424673 label:  -1\n",
      "i = 130, 9.21985815602837% done\n",
      "0:  0.08983030672397607 1:  0.08199974150354826 label:  -1\n",
      "0:  0.13903652756358245 1:  0.1325344403447931 label:  -1\n",
      "0:  0.09568556488735423 1:  0.062094076359107846 label:  -1\n",
      "0:  0.06359936352709557 1:  0.14563015809916038 label:  -1\n",
      "0:  0.1239873135922247 1:  0.20855565314106353 label:  -1\n",
      "0:  0.08769029747177982 1:  0.13640709755530808 label:  -1\n",
      "0:  0.05895113118305703 1:  0.03250888505974497 label:  -1\n",
      "0:  0.06375992488440106 1:  0.044786722256885125 label:  -1\n",
      "0:  0.10846190369981869 1:  0.06932428948164987 label:  -1\n",
      "0:  0.1185030570852421 1:  0.09579596269841378 label:  -1\n",
      "i = 140, 9.929078014184396% done\n",
      "0:  0.06340001940309385 1:  0.039748152218933214 label:  -1\n",
      "0:  0.03603123170673476 1:  0.01849129953758374 label:  -1\n",
      "0:  0.07674390206017383 1:  0.1290639222660288 label:  -1\n",
      "0:  0.1319576531643451 1:  0.09864736063593296 label:  -1\n",
      "0:  0.15493568706363092 1:  0.0856691266439648 label:  -1\n",
      "0:  0.08957560076735313 1:  0.06541542019154593 label:  -1\n",
      "0:  0.20918600358146472 1:  0.11971893905969873 label:  -1\n",
      "0:  0.07613761005433314 1:  0.0539284257110134 label:  -1\n",
      "0:  0.1748241503193139 1:  0.09613035516815793 label:  -1\n",
      "0:  0.0748613503148164 1:  0.08316559106660455 label:  -1\n",
      "i = 150, 10.638297872340425% done\n",
      "0:  0.02340715475223765 1:  0.015685070560286587 label:  -1\n",
      "0:  0.054842792912247376 1:  0.04124844531512661 label:  -1\n",
      "0:  0.09994619186997677 1:  0.2048049646856465 label:  -1\n",
      "0:  0.2379196399159102 1:  0.19203016137661677 label:  -1\n",
      "0:  0.13699619148076214 1:  0.10403835981854904 label:  -1\n",
      "0:  0.12851647404717934 1:  0.2093374142558305 label:  -1\n",
      "0:  0.08194302028242884 1:  0.06416369284688057 label:  -1\n",
      "0:  0.06351628797217714 1:  0.03483611546114231 label:  -1\n",
      "0:  0.03788333285229883 1:  0.02698230985753251 label:  -1\n",
      "0:  0.08854852173652608 1:  0.11437303156438265 label:  -1\n",
      "i = 160, 11.347517730496454% done\n",
      "0:  0.13676643413087583 1:  0.09879197261663823 label:  -1\n",
      "0:  0.1491607088867961 1:  0.07478310627275651 label:  -1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "ignored",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-29-351a68ad352b>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0moneHotVector\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mzeros\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mllda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocas\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mj\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrimmedTestCorpus\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mtrimmedTestCorpus\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mllda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocas\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m             \u001b[0moneHotVector\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mllda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvocas\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrimmedTestCorpus\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mj\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m     \u001b[0;31m# Then, dot product that one-hot vector with the phi vector and see what the most likely topic is\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# for dev data (validation)\n",
    "for i in range(len(testSet)):\n",
    "    # Each column is a one-hot entry indicating the existence of a particular word from the vocab in the dataset\n",
    "    oneHotVector = np.zeros([1, len(llda.vocas)])\n",
    "    for j in range(len(trimmedTestCorpus[i][:])):\n",
    "        if trimmedTestCorpus[i][j] in llda.vocas:\n",
    "            oneHotVector[0, llda.vocas.index(trimmedTestCorpus[i][j])] = 1\n",
    "            \n",
    "    # Dot-product that one-hot vector with the phi vector and see what the most likely topic is\n",
    "    dotProductReal = np.dot(oneHotVector[0, :], phi[1, :])\n",
    "    dotProductFake = np.dot(oneHotVector[0, :], phi[2, :])\n",
    "    \n",
    "    #print(\"dot product with topic common: \" + str(dotProductCommon))\n",
    "    #print(\"dot product with topic real: \" + str(dotProductReal))\n",
    "    #print(\"dot product with topic fake: \" + str(dotProductFake))\n",
    "    print(\"0: \", dotProductReal,\"1: \", dotProductFake, 'label: ', testLabel[i])\n",
    "    if dotProductReal < dotProductFake and dotProductFake>0.1 :\n",
    "        expected = 1.0\n",
    "    else:\n",
    "        expected = 0.0\n",
    "        \n",
    "    pred_classes[i] = expected\n",
    "    #print(\"Actual label: \" + str(testSet[i, 1]))\n",
    "    \n",
    "    if expected == testLabel[i]:\n",
    "        totalPredictionsCorrect = totalPredictionsCorrect + 1\n",
    "    \n",
    "    if i % 10 == 0 and i > 0:\n",
    "        print(\"i = \" + str(i) + \", \" + str(100.*i/len(testSet)) + \"% done\")\n",
    "    \n",
    "    #print(\"\\n\\n\")\n",
    "\n",
    "#print(\"Testing Accuracy: \" + str((100.*totalPredictionsCorrect)/len(testSet)))\n",
    "#print(\"Matthews Correlation Coeff: \" + str(matthews_corrcoef(y_true=true_classes, y_pred=pred_classes)))\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "IGN5ZxtqpZ4U"
   },
   "outputs": [],
   "source": [
    "# for submit test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "id": "1a-G6YU6xHUP",
    "outputId": "b307e59a-1ee2-4564-9307-5b6b4fb0d7b4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "i = 10, 0.7092198581560284% done\n",
      "i = 20, 1.4184397163120568% done\n",
      "i = 30, 2.127659574468085% done\n",
      "i = 40, 2.8368794326241136% done\n",
      "i = 50, 3.5460992907801416% done\n",
      "i = 60, 4.25531914893617% done\n",
      "i = 70, 4.964539007092198% done\n",
      "i = 80, 5.673758865248227% done\n",
      "i = 90, 6.382978723404255% done\n",
      "i = 100, 7.092198581560283% done\n",
      "i = 110, 7.801418439716312% done\n",
      "i = 120, 8.51063829787234% done\n",
      "i = 130, 9.21985815602837% done\n",
      "i = 140, 9.929078014184396% done\n",
      "i = 150, 10.638297872340425% done\n",
      "i = 160, 11.347517730496454% done\n",
      "i = 170, 12.056737588652481% done\n",
      "i = 180, 12.76595744680851% done\n",
      "i = 190, 13.47517730496454% done\n",
      "i = 200, 14.184397163120567% done\n",
      "i = 210, 14.893617021276595% done\n",
      "i = 220, 15.602836879432624% done\n",
      "i = 230, 16.31205673758865% done\n",
      "i = 240, 17.02127659574468% done\n",
      "i = 250, 17.73049645390071% done\n",
      "i = 260, 18.43971631205674% done\n",
      "i = 270, 19.148936170212767% done\n",
      "i = 280, 19.858156028368793% done\n",
      "i = 290, 20.56737588652482% done\n",
      "i = 300, 21.27659574468085% done\n",
      "i = 310, 21.98581560283688% done\n",
      "i = 320, 22.69503546099291% done\n",
      "i = 330, 23.404255319148938% done\n",
      "i = 340, 24.113475177304963% done\n",
      "i = 350, 24.822695035460992% done\n",
      "i = 360, 25.53191489361702% done\n",
      "i = 370, 26.24113475177305% done\n",
      "i = 380, 26.95035460992908% done\n",
      "i = 390, 27.659574468085108% done\n",
      "i = 400, 28.368794326241133% done\n",
      "i = 410, 29.078014184397162% done\n",
      "i = 420, 29.78723404255319% done\n",
      "i = 430, 30.49645390070922% done\n",
      "i = 440, 31.20567375886525% done\n",
      "i = 450, 31.914893617021278% done\n",
      "i = 460, 32.6241134751773% done\n",
      "i = 470, 33.333333333333336% done\n",
      "i = 480, 34.04255319148936% done\n",
      "i = 490, 34.751773049645394% done\n",
      "i = 500, 35.46099290780142% done\n",
      "i = 510, 36.170212765957444% done\n",
      "i = 520, 36.87943262411348% done\n",
      "i = 530, 37.5886524822695% done\n",
      "i = 540, 38.297872340425535% done\n",
      "i = 550, 39.00709219858156% done\n",
      "i = 560, 39.716312056737586% done\n",
      "i = 570, 40.42553191489362% done\n",
      "i = 580, 41.13475177304964% done\n",
      "i = 590, 41.843971631205676% done\n",
      "i = 600, 42.5531914893617% done\n",
      "i = 610, 43.262411347517734% done\n",
      "i = 620, 43.97163120567376% done\n",
      "i = 630, 44.680851063829785% done\n",
      "i = 640, 45.39007092198582% done\n",
      "i = 650, 46.09929078014184% done\n",
      "i = 660, 46.808510638297875% done\n",
      "i = 670, 47.5177304964539% done\n",
      "i = 680, 48.226950354609926% done\n",
      "i = 690, 48.93617021276596% done\n",
      "i = 700, 49.645390070921984% done\n",
      "i = 710, 50.354609929078016% done\n",
      "i = 720, 51.06382978723404% done\n",
      "i = 730, 51.773049645390074% done\n",
      "i = 740, 52.4822695035461% done\n",
      "i = 750, 53.191489361702125% done\n",
      "i = 760, 53.90070921985816% done\n",
      "i = 770, 54.60992907801418% done\n",
      "i = 780, 55.319148936170215% done\n",
      "i = 790, 56.02836879432624% done\n",
      "i = 800, 56.737588652482266% done\n",
      "i = 810, 57.4468085106383% done\n",
      "i = 820, 58.156028368794324% done\n",
      "i = 830, 58.86524822695036% done\n",
      "i = 840, 59.57446808510638% done\n",
      "i = 850, 60.283687943262414% done\n",
      "i = 860, 60.99290780141844% done\n",
      "i = 870, 61.702127659574465% done\n",
      "i = 880, 62.4113475177305% done\n",
      "i = 890, 63.12056737588652% done\n",
      "i = 900, 63.829787234042556% done\n",
      "i = 910, 64.53900709219859% done\n",
      "i = 920, 65.2482269503546% done\n",
      "i = 930, 65.95744680851064% done\n",
      "i = 940, 66.66666666666667% done\n",
      "i = 950, 67.37588652482269% done\n",
      "i = 960, 68.08510638297872% done\n",
      "i = 970, 68.79432624113475% done\n",
      "i = 980, 69.50354609929079% done\n",
      "i = 990, 70.2127659574468% done\n",
      "i = 1000, 70.92198581560284% done\n",
      "i = 1010, 71.63120567375887% done\n",
      "i = 1020, 72.34042553191489% done\n",
      "i = 1030, 73.04964539007092% done\n",
      "i = 1040, 73.75886524822695% done\n",
      "i = 1050, 74.46808510638297% done\n",
      "i = 1060, 75.177304964539% done\n",
      "i = 1070, 75.88652482269504% done\n",
      "i = 1080, 76.59574468085107% done\n",
      "i = 1090, 77.30496453900709% done\n",
      "i = 1100, 78.01418439716312% done\n",
      "i = 1110, 78.72340425531915% done\n",
      "i = 1120, 79.43262411347517% done\n",
      "i = 1130, 80.1418439716312% done\n",
      "i = 1140, 80.85106382978724% done\n",
      "i = 1150, 81.56028368794327% done\n",
      "i = 1160, 82.26950354609929% done\n",
      "i = 1170, 82.97872340425532% done\n",
      "i = 1180, 83.68794326241135% done\n",
      "i = 1190, 84.39716312056737% done\n",
      "i = 1200, 85.1063829787234% done\n",
      "i = 1210, 85.81560283687944% done\n",
      "i = 1220, 86.52482269503547% done\n",
      "i = 1230, 87.23404255319149% done\n",
      "i = 1240, 87.94326241134752% done\n",
      "i = 1250, 88.65248226950355% done\n",
      "i = 1260, 89.36170212765957% done\n",
      "i = 1270, 90.0709219858156% done\n",
      "i = 1280, 90.78014184397163% done\n",
      "i = 1290, 91.48936170212765% done\n",
      "i = 1300, 92.19858156028369% done\n",
      "i = 1310, 92.90780141843972% done\n",
      "i = 1320, 93.61702127659575% done\n",
      "i = 1330, 94.32624113475177% done\n",
      "i = 1340, 95.0354609929078% done\n",
      "i = 1350, 95.74468085106383% done\n",
      "i = 1360, 96.45390070921985% done\n",
      "i = 1370, 97.16312056737588% done\n",
      "i = 1380, 97.87234042553192% done\n",
      "i = 1390, 98.58156028368795% done\n",
      "i = 1400, 99.29078014184397% done\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(testSet)):\n",
    "    oneHotVector = np.zeros([1, len(llda.vocas)])\n",
    "    for j in range(len(trimmedTestCorpus[i][:])):\n",
    "        if trimmedTestCorpus[i][j] in llda.vocas:\n",
    "            oneHotVector[0, llda.vocas.index(trimmedTestCorpus[i][j])] = 1\n",
    "\n",
    "    dotProductReal = np.dot(oneHotVector[0, :], phi[1, :])\n",
    "    dotProductFake = np.dot(oneHotVector[0, :], phi[2, :])\n",
    "\n",
    "    if dotProductReal < dotProductFake:\n",
    "        expected = 1.0\n",
    "    else:\n",
    "        expected = 0.0\n",
    "        \n",
    "    pred_classes[i] = expected\n",
    "\n",
    "    if i % 10 == 0 and i > 0:\n",
    "        print(\"i = \" + str(i) + \", \" + str(100.*i/len(testSet)) + \"% done\")\n",
    "    \n",
    "    #print(\"\\n\\n\")\n",
    "#print(\"Testing Accuracy: \" + str((100.*totalPredictionsCorrect)/len(testSet)))\n",
    "#print(\"Matthews Correlation Coeff: \" + str(matthews_corrcoef(y_true=true_classes, y_pred=pred_classes)))\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "leJmTr6BxHXr"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "\n",
    "result = classification_report(true_classes, pred_classes) \n",
    "print ('\\n clasification report:\\n', result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 252
    },
    "id": "t6vADVfqxHbA",
    "outputId": "42dbd2be-f239-4923-f6ee-06e08b59c178"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      prediction\n",
      "0              1\n",
      "1              0\n",
      "2              0\n",
      "3              0\n",
      "4              1\n",
      "...          ...\n",
      "1405           1\n",
      "1406           0\n",
      "1407           1\n",
      "1408           1\n",
      "1409           0\n",
      "\n",
      "[1410 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "pred_classes = [int(x) for x in pred_classes]\n",
    "result = np.array(pred_classes)\n",
    "submission_df = pd.DataFrame(data=result, columns=['prediction'])\n",
    "print(submission_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "lPvOhwxdxHd8"
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from google.colab import files\n",
    "#{\"test-0\": {\"label\": 0}, \"test-1\": {\"label\": 0}, ... ,}\n",
    "data = {}\n",
    "df = submission_df['prediction']\n",
    "for idx, rows in enumerate(df):\n",
    "    label = {}\n",
    "    _id = 'test-'+str(idx)\n",
    "    label[\"label\"] = rows\n",
    "    data[_id] = label\n",
    "\n",
    "jsonfilename = 'test-output.json'\n",
    "with open(jsonfilename, 'w') as jsonFile:\n",
    "    jsonFile.write(json.dumps(data))\n",
    "\n",
    "files.download('test-output.json')"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "name": "LLDA2.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}