CH04 Naive Bayes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CH4  Naive Bayes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## MLiA"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 文档[句子[词条]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 243,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def loadDataSet():\n",
    "    postingList = [['my','dog','has','flea','problems','help','please'],\n",
    "                  ['maybe','not','take','him','to','dog','park','stupid'],\n",
    "                  ['my','dalmation','is','so','cute','I','love','him'],\n",
    "                  ['stop','posting','stupid','worthles','garbage','to','stop','him'],\n",
    "                  ['quit','buying','worthless','dog','food','stupid']]\n",
    "    classVec = [0,1,0,1,0,1]\n",
    "    return postingList, classVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "postingList,classVec = loadDataSet()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 文档->词表[List]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 244,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def createVocabList(dataSet):\n",
    "    return list(set([y for x in dataSet for y in x]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "myVocabList = createVocabList(postingList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 250,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['so',\n",
       " 'posting',\n",
       " 'maybe',\n",
       " 'dalmation',\n",
       " 'food',\n",
       " 'help',\n",
       " 'quit',\n",
       " 'take',\n",
       " 'buying',\n",
       " 'please',\n",
       " 'park',\n",
       " 'is',\n",
       " 'problems',\n",
       " 'garbage',\n",
       " 'stop',\n",
       " 'cute',\n",
       " 'worthles',\n",
       " 'not',\n",
       " 'to',\n",
       " 'stupid',\n",
       " 'worthless',\n",
       " 'love',\n",
       " 'I',\n",
       " 'has',\n",
       " 'flea',\n",
       " 'him',\n",
       " 'dog',\n",
       " 'my']"
      ]
     },
     "execution_count": 250,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "myVocabList"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 句子+词表->词向量[List]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 240,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def setOfWords2Vec(vocabList, inputSet):\n",
    "    returnVec = [0]*len(vocabList)\n",
    "    for word in inputSet:\n",
    "        if word in vocabList:\n",
    "            returnVec[vocabList.index(word)] = 1\n",
    "        else:\n",
    "            print(\"the word: %s is not in my Vocabulary!\" % word)\n",
    "    return returnVec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 251,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 0,\n",
       " 1,\n",
       " 1,\n",
       " 0,\n",
       " 1,\n",
       " 1]"
      ]
     },
     "execution_count": 251,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "setOfWords2Vec(myVocabList,postingList[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 词向量->概率"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Set of Word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "p(w/ci),p(ci)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 257,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def trainNB0(trainMatrix,trainCategory):\n",
    "    numTrainDocs = len(trainMatrix) # 句子，样本数\n",
    "    numWords = len(trainMatrix[0])  # 词向量长度\n",
    "    pAbusive = sum(trainCategory)/float(numTrainDocs) # 类别为1的样本占比\n",
    "    p0Num = np.ones(numWords);p1Num = np.ones(numWords)\n",
    "    p0Denom = 2.0; p1Denom = 2.0\n",
    "    for i in range(numTrainDocs):\n",
    "        if trainCategory[i] == 1:\n",
    "            p1Num += trainMatrix[i]        # 向量：样本汇总，所有样本加到一个词向量长度的向量\n",
    "            p1Denom += sum(trainMatrix[i]) # 标量：所有单词计数\n",
    "        else:\n",
    "            p0Num += trainMatrix[i]\n",
    "            p0Denom += sum(trainMatrix[i])\n",
    "    p1Vect = np.log(p1Num/p1Denom)   # 向量，训练数据中归为1类的样本统计出的，词向量中每个元素出现的概率\n",
    "    p0Vect = np.log(p0Num/p0Denom)   # \n",
    "    return p0Vect, p1Vect, pAbusive"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def classifyNB(vec2Classify, p0Vec, p1Vec, pClass1):\n",
    "    p1 = sum(vec2Classify*p1Vec) + np.log(pClass1)\n",
    "    p0 = sum(vec2Classify*p0Vec) + np.log(1-pClass1)\n",
    "    return 1 if p1 > p0 else 0 # 这里可以再处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testingNB():\n",
    "    listOfPosts, listClasses = loadDataSet()\n",
    "    myVocabList = createVocabList(listOfPosts)\n",
    "    trainMat = []\n",
    "    for postinDoc in listOfPosts:\n",
    "        trainMat.append(setOfWords2Vec(myVocabList,postinDoc))\n",
    "    p0V,p1V,pAb = trainNB0(np.array(trainMat),np.array(listClasses))\n",
    "    testEntry = [\"love\",\"my\",\"dalmation\"]\n",
    "    thisDoc = np.array(setOfWords2Vec(myVocabList,testEntry))\n",
    "    print(testEntry,\"classified as: \",classifyNB(thisDoc,p0Vec=p0V,p1Vec=p1V,pClass1=pAb))\n",
    "    testEntry = [\"stupid\",\"garbage\"]\n",
    "    thisDoc = np.array(setOfWords2Vec(myVocabList,testEntry))\n",
    "    print(testEntry,\"classified as: \",classifyNB(thisDoc,p0Vec=p0V,p1Vec=p1V,pClass1=pAb))    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['love', 'my', 'dalmation'] classified as:  0\n",
      "['stupid', 'garbage'] classified as:  1\n"
     ]
    }
   ],
   "source": [
    "testingNB()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Bag of Word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def bagOfWords2VecMN(vocabList,inputSet):\n",
    "    returnVec = [0]*len(vocabList)\n",
    "    for word in inputSet:\n",
    "        if word in vocabList:\n",
    "            returnVec[vocabList.index(word)] += 1\n",
    "        else:\n",
    "            print(\"the word: %s is not in my Vocabulary!\" % word)\n",
    "    return returnVec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX1 : Spam"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def textParse(bigString):\n",
    "    import re\n",
    "    listOfTokens = re.split(r'\\W*',bigString)\n",
    "    return [tok.lower() for tok in listOfTokens if len(tok) > 2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import math\n",
    "def spamTest():\n",
    "    docList=[];classList=[];fullText=[]\n",
    "    for i in list(range(1,26)):\n",
    "        wordList = textParse(open('./Data/CH4/email/spam/%d.txt' % i,encoding=\"latin_1\").read())\n",
    "        docList.append(wordList)\n",
    "        fullText.extend(wordList)\n",
    "        classList.append(1)\n",
    "        wordList = textParse(open('./Data/CH4/email/ham/%d.txt' % i,encoding=\"latin_1\").read())\n",
    "        docList.append(wordList)\n",
    "        fullText.extend(wordList)\n",
    "        classList.append(0)\n",
    "    vocabList = createVocabList(docList)\n",
    "    trainingSet = list(range(len(wordList)));testSet=[]\n",
    "    for i in list(range(10)):\n",
    "        randIndex = int(np.random.uniform(0,len(trainingSet)))\n",
    "        testSet.append(trainingSet[randIndex])\n",
    "        del(trainingSet[randIndex])\n",
    "    trainMat = [];trainClasses = []\n",
    "    for docIndex in trainingSet:\n",
    "        trainMat.append(bagOfWords2VecMN(vocabList,docList[docIndex]))\n",
    "        trainClasses.append(classList[docIndex])\n",
    "    p0V,p1V,pSpam = trainNB0(np.array(trainMat),np.array(trainClasses))\n",
    "    errorCount = 0\n",
    "    for docIndex in testSet:\n",
    "        wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])\n",
    "        if classifyNB(np.array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:\n",
    "            errorCount += 1\n",
    "    print(\"the error rate is : \", errorCount/len(testSet))\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the error rate is :  0.6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: RuntimeWarning: divide by zero encountered in log\n",
      "  \n"
     ]
    }
   ],
   "source": [
    "spamTest()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX2 : Local Word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 231,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calcMostFreq(vocabList,fullText):\n",
    "    import operator\n",
    "    freqDict = {}\n",
    "    for token in vocabList:\n",
    "        freqDict[token] = fullText.count(token)\n",
    "#     print(freqDict)\n",
    "    sortedFreq = sorted(freqDict.items(),key=lambda x:x[1],reverse=True)\n",
    "    return sortedFreq[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def localWords(feed1,feed0):\n",
    "    import feedparser\n",
    "    docList=[]; classList = []; fullText =[]\n",
    "    minLen = min(len(feed1['entries']),len(feed0['entries']))\n",
    "    for i in list(range(minLen)):\n",
    "        wordList = textParse(feed1['entries'][i]['summary'])\n",
    "        docList.append(wordList)\n",
    "        fullText.extend(wordList)\n",
    "        classList.append(1) #NY is class 1\n",
    "        wordList = textParse(feed0['entries'][i]['summary'])\n",
    "        docList.append(wordList)\n",
    "        fullText.extend(wordList)\n",
    "        classList.append(0)\n",
    "    vocabList = createVocabList(docList)#create vocabulary\n",
    "    print(len(vocabList))\n",
    "    top30Words = calcMostFreq(vocabList,fullText)   #remove top 30 words\n",
    "    for pairW in top30Words:\n",
    "        if pairW[0] in vocabList: vocabList.remove(pairW[0])\n",
    "            \n",
    "    # https://www.ranks.nl/stopwords\n",
    "    with open(\"./Data/CH4/stopwords.txt\") as f:\n",
    "        stopwords = f.readlines()\n",
    "    stopwords = [x.strip() for x in stopwords]\n",
    "    for pairW in stopwords:\n",
    "        if pairW in vocabList: vocabList.remove(pairW)\n",
    "\n",
    "    print(len(vocabList))\n",
    "    trainingSet = list(range(2*minLen)); testSet=[]           #create test set\n",
    "    for i in list(range(20)):\n",
    "        randIndex = int(np.random.uniform(0,len(trainingSet)))\n",
    "        testSet.append(trainingSet[randIndex])\n",
    "        del(trainingSet[randIndex])  \n",
    "    trainMat=[]; trainClasses = []\n",
    "    for docIndex in trainingSet:#train the classifier (get probs) trainNB0\n",
    "        trainMat.append(bagOfWords2VecMN(vocabList, docList[docIndex]))\n",
    "        trainClasses.append(classList[docIndex])\n",
    "    p0V,p1V,pSpam = trainNB0(np.array(trainMat),np.array(trainClasses))\n",
    "    errorCount = 0\n",
    "    for docIndex in testSet:        #classify the remaining items\n",
    "        wordVector = bagOfWords2VecMN(vocabList, docList[docIndex])\n",
    "        if classifyNB(np.array(wordVector),p0V,p1V,pSpam) != classList[docIndex]:\n",
    "            errorCount += 1\n",
    "    print('the error rate is: ',float(errorCount)/len(testSet))\n",
    "    return vocabList,p0V,p1V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 233,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import feedparser\n",
    "ny = feedparser.parse(\"https://newyork.craigslist.org/search/sof?format=rss\")\n",
    "sf = feedparser.parse(\"https://sfbay.craigslist.org/search/sof?format=rss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "692\n",
      "632\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: all is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: only is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: doing is not in my Vocabulary!\n",
      "the word: would is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: your is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: has is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: was is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: all is not in my Vocabulary!\n",
      "the word: over is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: its is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: being is not in my Vocabulary!\n",
      "the word: most is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: what is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: could is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: their is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: when is not in my Vocabulary!\n",
      "the word: your is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: not is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: some is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: other is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: out is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: out is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: where is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: some is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: over is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: after is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: after is not in my Vocabulary!\n",
      "the word: off is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: this is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    },
    {
     "ename": "NameError",
     "evalue": "name 'trainNB0' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-253-b969a1bd21f7>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mvocabList\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpSF\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpNY\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlocalWords\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mny\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0msf\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-232-867210267b86>\u001b[0m in \u001b[0;36mlocalWords\u001b[1;34m(feed1, feed0)\u001b[0m\n\u001b[0;32m     35\u001b[0m         \u001b[0mtrainMat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbagOfWords2VecMN\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvocabList\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdocList\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdocIndex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[0mtrainClasses\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mclassList\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mdocIndex\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m     \u001b[0mp0V\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mp1V\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mpSpam\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtrainNB0\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainMat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtrainClasses\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m     \u001b[0merrorCount\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m     \u001b[1;32mfor\u001b[0m \u001b[0mdocIndex\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtestSet\u001b[0m\u001b[1;33m:\u001b[0m        \u001b[1;31m#classify the remaining items\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mNameError\u001b[0m: name 'trainNB0' is not defined"
     ]
    }
   ],
   "source": [
    "vocabList,pSF,pNY = localWords(ny,sf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 254,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# https://www.ranks.nl/stopwords\n",
    "with open(\"./Data/CH4//stopwords.txt\") as f:\n",
    "    stopwords = f.readlines()\n",
    "stopwords = [x.strip() for x in stopwords]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 255,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def getTopWords(ny,sf):\n",
    "    import operator\n",
    "    vocabList,p0V,p1V=localWords(ny,sf)\n",
    "    topNY=[]; topSF=[]\n",
    "    for i in list(range(len(p0V))):\n",
    "        if p0V[i] > -6.0 : topSF.append((vocabList[i],p0V[i]))\n",
    "        if p1V[i] > -6.0 : topNY.append((vocabList[i],p1V[i]))\n",
    "    sortedSF = sorted(topSF, key=lambda pair: pair[1], reverse=True)\n",
    "    print(\"SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**\")\n",
    "    for item in sortedSF:\n",
    "        print (item[0])\n",
    "    sortedNY = sorted(topNY, key=lambda pair: pair[1], reverse=True)\n",
    "    print(\"NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**\")\n",
    "    for item in sortedNY:\n",
    "        print (item[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\Anaconda3\\lib\\re.py:212: FutureWarning: split() requires a non-empty pattern match.\n",
      "  return _compile(pattern, flags).split(string, maxsplit)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "692\n",
      "632\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: all is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: all is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: being is not in my Vocabulary!\n",
      "the word: most is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: what is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: could is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: doing is not in my Vocabulary!\n",
      "the word: what is not in my Vocabulary!\n",
      "the word: they is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: doing is not in my Vocabulary!\n",
      "the word: would is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: your is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: has is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: all is not in my Vocabulary!\n",
      "the word: over is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: its is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: they is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: some is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: being is not in my Vocabulary!\n",
      "the word: most is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: what is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: could is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: being is not in my Vocabulary!\n",
      "the word: most is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: what is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: could is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: doing is not in my Vocabulary!\n",
      "the word: would is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: your is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: when is not in my Vocabulary!\n",
      "the word: your is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: not is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: other is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: under is not in my Vocabulary!\n",
      "the word: has is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: out is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: some is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: over is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: after is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: after is not in my Vocabulary!\n",
      "the word: off is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: out is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: has is not in my Vocabulary!\n",
      "the word: some is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: has is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: time is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: most is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: was is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: development is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: about is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: developer is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: this is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: experience is not in my Vocabulary!\n",
      "the word: you is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: below is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: your is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: how is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: some is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: other is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: work is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: data is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: who is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: only is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: services is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: software is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: its is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: are is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: with is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: that is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: looking is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: applications is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: their is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: analyst is not in my Vocabulary!\n",
      "the word: for is not in my Vocabulary!\n",
      "the word: more is not in my Vocabulary!\n",
      "the word: where is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: seeks is not in my Vocabulary!\n",
      "the word: full is not in my Vocabulary!\n",
      "the word: web is not in my Vocabulary!\n",
      "the word: design is not in my Vocabulary!\n",
      "the word: test is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: out is not in my Vocabulary!\n",
      "the word: new is not in my Vocabulary!\n",
      "the word: code is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: from is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: tech is not in my Vocabulary!\n",
      "the word: company is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the word: their is not in my Vocabulary!\n",
      "the word: their is not in my Vocabulary!\n",
      "the word: and is not in my Vocabulary!\n",
      "the word: their is not in my Vocabulary!\n",
      "the word: our is not in my Vocabulary!\n",
      "the word: have is not in my Vocabulary!\n",
      "the word: the is not in my Vocabulary!\n",
      "the error rate is:  0.3\n",
      "SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**SF**\n",
      "mobile\n",
      "games\n",
      "team\n",
      "generation\n",
      "california\n",
      "startup\n",
      "building\n",
      "fast\n",
      "growing\n",
      "oconus\n",
      "hosting\n",
      "ship\n",
      "uav\n",
      "daily\n",
      "will\n",
      "core\n",
      "millions\n",
      "bay\n",
      "microsoft\n",
      "native\n",
      "saas\n",
      "position\n",
      "online\n",
      "draw\n",
      "area\n",
      "love\n",
      "play\n",
      "game\n",
      "engineering\n",
      "enjoy\n",
      "hated\n",
      "open\n",
      "react\n",
      "rep\n",
      "connect\n",
      "paced\n",
      "technology\n",
      "app\n",
      "vehicles\n",
      "award\n",
      "source\n",
      "database\n",
      "nav\n",
      "loved\n",
      "driven\n",
      "inc\n",
      "contribute\n",
      "field\n",
      "magento\n",
      "craigslist\n",
      "various\n",
      "join\n",
      "mojo\n",
      "erp\n",
      "projects\n",
      "world\n",
      "end\n",
      "casino\n",
      "dynamics\n",
      "people\n",
      "stratus\n",
      "unity\n",
      "see\n",
      "critical\n",
      "level\n",
      "automa\n",
      "summer\n",
      "platforms\n",
      "members\n",
      "intermediate\n",
      "impactful\n",
      "makes\n",
      "professional\n",
      "laid\n",
      "ready\n",
      "fish\n",
      "role\n",
      "things\n",
      "school\n",
      "amp\n",
      "growth\n",
      "experiences\n",
      "magemojo\n",
      "marketing\n",
      "across\n",
      "management\n",
      "lead\n",
      "thecoderschool\n",
      "carolina\n",
      "party\n",
      "previous\n",
      "santa\n",
      "become\n",
      "meetups\n",
      "working\n",
      "china\n",
      "leading\n",
      "picollo\n",
      "let\n",
      "plenty\n",
      "talent\n",
      "learning\n",
      "held\n",
      "road\n",
      "built\n",
      "net\n",
      "gain\n",
      "vallejo\n",
      "car\n",
      "experi\n",
      "driverless\n",
      "devops\n",
      "operations\n",
      "massive\n",
      "tops\n",
      "roll\n",
      "potential\n",
      "unmanned\n",
      "want\n",
      "advanced\n",
      "cloud\n",
      "problems\n",
      "implement\n",
      "north\n",
      "coding\n",
      "exciting\n",
      "containerized\n",
      "number\n",
      "coast\n",
      "young\n",
      "barbara\n",
      "engine\n",
      "keep\n",
      "searching\n",
      "benefit\n",
      "concepts\n",
      "closely\n",
      "current\n",
      "ever\n",
      "aware\n",
      "kingdom\n",
      "among\n",
      "client\n",
      "bruce\n",
      "systems\n",
      "masonbruce\n",
      "united\n",
      "positions\n",
      "thousands\n",
      "guys\n",
      "support\n",
      "months\n",
      "big\n",
      "talented\n",
      "intern\n",
      "geog\n",
      "fill\n",
      "several\n",
      "usfs\n",
      "dependent\n",
      "duties\n",
      "maintain\n",
      "next\n",
      "android\n",
      "platform\n",
      "publishing\n",
      "www\n",
      "francisco\n",
      "headquartered\n",
      "techexcel\n",
      "fsr\n",
      "self\n",
      "back\n",
      "eventually\n",
      "perform\n",
      "computers\n",
      "places\n",
      "databases\n",
      "behalf\n",
      "com\n",
      "profile\n",
      "group\n",
      "analytical\n",
      "environment\n",
      "drop\n",
      "email\n",
      "teach\n",
      "enviro\n",
      "edge\n",
      "workshops\n",
      "dynamic\n",
      "user\n",
      "idealistic\n",
      "stores\n",
      "requirements\n",
      "works\n",
      "safely\n",
      "teaching\n",
      "gals\n",
      "program\n",
      "first\n",
      "aws\n",
      "best\n",
      "engineer\n",
      "autonomous\n",
      "care\n",
      "developing\n",
      "resource\n",
      "tools\n",
      "consultants\n",
      "updates\n",
      "mason\n",
      "geospatial\n",
      "cto\n",
      "military\n",
      "summary\n",
      "entry\n",
      "get\n",
      "oversight\n",
      "developers\n",
      "founder\n",
      "san\n",
      "future\n",
      "spatial\n",
      "offices\n",
      "please\n",
      "small\n",
      "features\n",
      "free\n",
      "scientists\n",
      "established\n",
      "girard\n",
      "talk\n",
      "paas\n",
      "said\n",
      "creates\n",
      "bright\n",
      "navigating\n",
      "social\n",
      "apps\n",
      "stake\n",
      "arizona\n",
      "locations\n",
      "cutting\n",
      "natural\n",
      "overview\n",
      "superior\n",
      "holders\n",
      "contract\n",
      "site\n",
      "charts\n",
      "going\n",
      "instructor\n",
      "provider\n",
      "winning\n",
      "remotely\n",
      "come\n",
      "arcturus\n",
      "opportunities\n",
      "hundreds\n",
      "locate\n",
      "including\n",
      "franchise\n",
      "fulfill\n",
      "engineers\n",
      "relational\n",
      "netline\n",
      "candidates\n",
      "privately\n",
      "job\n",
      "ios\n",
      "solutions\n",
      "real\n",
      "develop\n",
      "kids\n",
      "michigan\n",
      "motivated\n",
      "need\n",
      "servic\n",
      "ultimate\n",
      "help\n",
      "mini\n",
      "fragrances\n",
      "year\n",
      "min\n",
      "largest\n",
      "run\n",
      "fortune\n",
      "preferred\n",
      "enterprises\n",
      "video\n",
      "exposing\n",
      "description\n",
      "bachelor\n",
      "architecture\n",
      "analysis\n",
      "medical\n",
      "commerce\n",
      "unlocking\n",
      "electrical\n",
      "top\n",
      "improvements\n",
      "models\n",
      "leetcode\n",
      "aromatherapy\n",
      "iterate\n",
      "sql\n",
      "excel\n",
      "placement\n",
      "hadoop\n",
      "relationship\n",
      "short\n",
      "soundpruf\n",
      "node\n",
      "indoors\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trackingtag\n",
      "solving\n",
      "seeking\n",
      "cold\n",
      "accounting\n",
      "internship\n",
      "general\n",
      "now\n",
      "shown\n",
      "tester\n",
      "can\n",
      "graphics\n",
      "combo\n",
      "mateo\n",
      "25011\n",
      "rds\n",
      "technical\n",
      "clerk\n",
      "independent\n",
      "jersey\n",
      "architectures\n",
      "puzzle\n",
      "dependance\n",
      "content\n",
      "knowledge\n",
      "queens\n",
      "reduce\n",
      "successful\n",
      "train\n",
      "library\n",
      "leave\n",
      "must\n",
      "find\n",
      "providing\n",
      "interested\n",
      "creators\n",
      "jobs\n",
      "recovery\n",
      "range\n",
      "buy\n",
      "main\n",
      "programming\n",
      "python\n",
      "softwa\n",
      "proper\n",
      "computer\n",
      "withgoogle\n",
      "making\n",
      "jupiter\n",
      "electronic\n",
      "years\n",
      "fonts\n",
      "h1b\n",
      "presented\n",
      "consulting\n",
      "use\n",
      "addition\n",
      "devices\n",
      "specialist\n",
      "products\n",
      "benefits\n",
      "meet\n",
      "event\n",
      "document\n",
      "youtube\n",
      "era\n",
      "quality\n",
      "optical\n",
      "genuine\n",
      "living\n",
      "resume\n",
      "wbe\n",
      "overcoming\n",
      "imagine\n",
      "present\n",
      "tha\n",
      "blockchain\n",
      "meaningful\n",
      "demo\n",
      "planning\n",
      "business\n",
      "coordinator\n",
      "interest\n",
      "family\n",
      "grow\n",
      "island\n",
      "expert\n",
      "intervention\n",
      "administration\n",
      "human\n",
      "ads\n",
      "digital\n",
      "investigation\n",
      "firm\n",
      "graphql\n",
      "life\n",
      "survivors\n",
      "sick\n",
      "location\n",
      "know\n",
      "make\n",
      "excell\n",
      "challenging\n",
      "project\n",
      "information\n",
      "one\n",
      "disability\n",
      "title\n",
      "hedge\n",
      "class\n",
      "beauty\n",
      "tired\n",
      "students\n",
      "matlab\n",
      "beach\n",
      "thinker\n",
      "results\n",
      "delight\n",
      "tra\n",
      "100\n",
      "long\n",
      "background\n",
      "stack\n",
      "abuse\n",
      "expand\n",
      "crisis\n",
      "apply\n",
      "excited\n",
      "stuck\n",
      "learn\n",
      "provide\n",
      "science\n",
      "hired\n",
      "inventory\n",
      "salary\n",
      "part\n",
      "craftsman\n",
      "certified\n",
      "call\n",
      "department\n",
      "located\n",
      "securities\n",
      "7299u\n",
      "progress\n",
      "player\n",
      "skincare\n",
      "wide\n",
      "fully\n",
      "passionate\n",
      "training\n",
      "apis\n",
      "supportive\n",
      "interes\n",
      "solid\n",
      "technologies\n",
      "women\n",
      "berkeley\n",
      "skills\n",
      "regular\n",
      "strong\n",
      "package\n",
      "imports\n",
      "preferably\n",
      "public\n",
      "institutional\n",
      "website\n",
      "based\n",
      "dream\n",
      "just\n",
      "billion\n",
      "media\n",
      "var\n",
      "optimize\n",
      "accreditation\n",
      "globally\n",
      "watch\n",
      "remote\n",
      "p_aaaaaacaaadpa\n",
      "ranked\n",
      "name\n",
      "deploy\n",
      "senior\n",
      "link\n",
      "take\n",
      "extractions\n",
      "per\n",
      "temporary\n",
      "insurance\n",
      "experienced\n",
      "quickbooks\n",
      "hire\n",
      "course\n",
      "report\n",
      "coverage\n",
      "violence\n",
      "hour\n",
      "view\n",
      "either\n",
      "financial\n",
      "least\n",
      "designers\n",
      "large\n",
      "yrs\n",
      "brand\n",
      "excellent\n",
      "sparq\n",
      "agency\n",
      "quantitative\n",
      "analysts\n",
      "500\n",
      "creative\n",
      "landed\n",
      "renaissance\n",
      "domestic\n",
      "health\n",
      "organized\n",
      "county\n",
      "responsibilities\n",
      "enhance\n",
      "dental\n",
      "aperia\n",
      "term\n",
      "upgrade\n",
      "ticketing\n",
      "https\n",
      "qualifications\n",
      "templates\n",
      "debug\n",
      "investment\n",
      "multiple\n",
      "upload\n",
      "consideration\n",
      "profitability\n",
      "already\n",
      "mssql\n",
      "minority\n",
      "mission\n",
      "individual\n",
      "themes\n",
      "access\n",
      "proje\n",
      "city\n",
      "plus\n",
      "possess\n",
      "action\n",
      "think\n",
      "companies\n",
      "includes\n",
      "imaging\n",
      "ideal\n",
      "day\n",
      "person\n",
      "major\n",
      "every\n",
      "cora\n",
      "gener\n",
      "mappings\n",
      "fragrance\n",
      "customers\n",
      "coverag\n",
      "users\n",
      "land\n",
      "candles\n",
      "constfuct\n",
      "java\n",
      "challenges\n",
      "yet\n",
      "etc\n",
      "ruby\n",
      "programmer\n",
      "modernizes\n",
      "swift\n",
      "office\n",
      "interactive\n",
      "bugs\n",
      "2014\n",
      "physics\n",
      "minutes\n",
      "efficiency\n",
      "introduction\n",
      "aperiatechcom\n",
      "fund\n",
      "existing\n",
      "backend\n",
      "progressive\n",
      "req\n",
      "comm\n",
      "math\n",
      "campus\n",
      "enterprise\n",
      "suggest\n",
      "serverless\n",
      "created\n",
      "friends\n",
      "minimum\n",
      "community\n",
      "improve\n",
      "nvwbswx6\n",
      "playing\n",
      "around\n",
      "conversions\n",
      "offer\n",
      "university\n",
      "virtualization\n",
      "music\n",
      "owned\n",
      "visa\n",
      "funding\n",
      "weather\n",
      "javascript\n",
      "directly\n",
      "mid\n",
      "york\n",
      "following\n",
      "shj5bh6vtu8\n",
      "move\n",
      "automation\n",
      "NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**NY**\n",
      "java\n",
      "senior\n",
      "engineer\n",
      "year\n",
      "indoors\n",
      "cold\n",
      "now\n",
      "leave\n",
      "jupiter\n",
      "meet\n",
      "family\n",
      "positions\n",
      "sick\n",
      "tired\n",
      "beach\n",
      "platform\n",
      "stuck\n",
      "science\n",
      "skills\n",
      "think\n",
      "minutes\n",
      "candidates\n",
      "friends\n",
      "weather\n",
      "move\n",
      "level\n",
      "hosting\n",
      "will\n",
      "jobs\n",
      "programming\n",
      "computer\n",
      "years\n",
      "use\n",
      "cloud\n",
      "business\n",
      "life\n",
      "love\n",
      "strong\n",
      "based\n",
      "take\n",
      "small\n",
      "magento\n",
      "mojo\n",
      "city\n",
      "ideal\n",
      "etc\n",
      "office\n",
      "stratus\n",
      "york\n",
      "need\n",
      "unity\n",
      "min\n",
      "fortune\n",
      "enterprises\n",
      "exposing\n",
      "bachelor\n",
      "architecture\n",
      "analysis\n",
      "intermediate\n",
      "top\n",
      "ready\n",
      "models\n",
      "role\n",
      "seeking\n",
      "graphics\n",
      "experiences\n",
      "magemojo\n",
      "independent\n",
      "jersey\n",
      "management\n",
      "dependance\n",
      "lead\n",
      "content\n",
      "queens\n",
      "reduce\n",
      "successful\n",
      "meetups\n",
      "must\n",
      "leading\n",
      "creators\n",
      "learning\n",
      "range\n",
      "buy\n",
      "built\n",
      "python\n",
      "softwa\n",
      "proper\n",
      "electronic\n",
      "native\n",
      "fonts\n",
      "devops\n",
      "h1b\n",
      "addition\n",
      "products\n",
      "advanced\n",
      "saas\n",
      "event\n",
      "containerized\n",
      "engine\n",
      "living\n",
      "wbe\n",
      "concepts\n",
      "interest\n",
      "online\n",
      "island\n",
      "human\n",
      "investigation\n",
      "thousands\n",
      "guys\n",
      "make\n",
      "excell\n",
      "project\n",
      "talented\n",
      "one\n",
      "game\n",
      "hedge\n",
      "engineering\n",
      "matlab\n",
      "thinker\n",
      "delight\n",
      "tra\n",
      "long\n",
      "background\n",
      "hired\n",
      "salary\n",
      "certified\n",
      "connect\n",
      "wide\n",
      "apis\n",
      "startup\n",
      "technologies\n",
      "women\n",
      "group\n",
      "analytical\n",
      "email\n",
      "teach\n",
      "preferably\n",
      "website\n",
      "just\n",
      "billion\n",
      "edge\n",
      "media\n",
      "workshops\n",
      "requirements\n",
      "teaching\n",
      "gals\n",
      "ranked\n",
      "deploy\n",
      "per\n",
      "experienced\n",
      "care\n",
      "hour\n",
      "either\n",
      "least\n",
      "designers\n",
      "yrs\n",
      "brand\n",
      "excellent\n",
      "sparq\n",
      "quantitative\n",
      "analysts\n",
      "500\n",
      "entry\n",
      "renaissance\n",
      "driven\n",
      "developers\n",
      "responsibilities\n",
      "inc\n",
      "ticketing\n",
      "templates\n",
      "please\n",
      "multiple\n",
      "minority\n",
      "mission\n",
      "join\n",
      "themes\n",
      "talk\n",
      "proje\n",
      "possess\n",
      "paas\n",
      "building\n",
      "social\n",
      "day\n",
      "major\n",
      "every\n",
      "customers\n",
      "yet\n",
      "ruby\n",
      "swift\n",
      "instructor\n",
      "remotely\n",
      "physics\n",
      "fund\n",
      "backend\n",
      "progressive\n",
      "req\n",
      "math\n",
      "ios\n",
      "minimum\n",
      "community\n",
      "playing\n",
      "around\n",
      "owned\n",
      "visa\n",
      "real\n",
      "funding\n",
      "people\n",
      "develop\n",
      "directly\n",
      "mid\n",
      "ultimate\n",
      "help\n",
      "automation\n",
      "fragrances\n",
      "largest\n",
      "run\n",
      "critical\n",
      "preferred\n",
      "automa\n",
      "video\n",
      "summer\n",
      "description\n",
      "platforms\n",
      "members\n",
      "medical\n",
      "commerce\n",
      "impactful\n",
      "unlocking\n",
      "electrical\n",
      "makes\n",
      "ship\n",
      "professional\n",
      "laid\n",
      "improvements\n",
      "fish\n",
      "leetcode\n",
      "aromatherapy\n",
      "iterate\n",
      "sql\n",
      "excel\n",
      "placement\n",
      "uav\n",
      "hadoop\n",
      "relationship\n",
      "short\n",
      "things\n",
      "soundpruf\n",
      "node\n",
      "trackingtag\n",
      "solving\n",
      "accounting\n",
      "internship\n",
      "general\n",
      "school\n",
      "shown\n",
      "tester\n",
      "can\n",
      "amp\n",
      "growth\n",
      "combo\n",
      "mateo\n",
      "generation\n",
      "25011\n",
      "rds\n",
      "daily\n",
      "technical\n",
      "clerk\n",
      "marketing\n",
      "architectures\n",
      "across\n",
      "puzzle\n",
      "thecoderschool\n",
      "carolina\n",
      "knowledge\n",
      "party\n",
      "previous\n",
      "santa\n",
      "become\n",
      "mobile\n",
      "core\n",
      "working\n",
      "china\n",
      "train\n",
      "library\n",
      "picollo\n",
      "let\n",
      "find\n",
      "providing\n",
      "plenty\n",
      "talent\n",
      "interested\n",
      "millions\n",
      "recovery\n",
      "held\n",
      "road\n",
      "main\n",
      "net\n",
      "bay\n",
      "gain\n",
      "vallejo\n",
      "car\n",
      "withgoogle\n",
      "experi\n",
      "making\n",
      "microsoft\n",
      "driverless\n",
      "presented\n",
      "consulting\n",
      "operations\n",
      "massive\n",
      "devices\n",
      "tops\n",
      "specialist\n",
      "roll\n",
      "potential\n",
      "unmanned\n",
      "want\n",
      "benefits\n",
      "problems\n",
      "implement\n",
      "north\n",
      "coding\n",
      "exciting\n",
      "document\n",
      "number\n",
      "coast\n",
      "young\n",
      "youtube\n",
      "era\n",
      "quality\n",
      "barbara\n",
      "optical\n",
      "genuine\n",
      "resume\n",
      "keep\n",
      "overcoming\n",
      "imagine\n",
      "searching\n",
      "present\n",
      "tha\n",
      "benefit\n",
      "blockchain\n",
      "closely\n",
      "position\n",
      "current\n",
      "ever\n",
      "aware\n",
      "meaningful\n",
      "demo\n",
      "planning\n",
      "coordinator\n",
      "kingdom\n",
      "among\n",
      "grow\n",
      "client\n",
      "expert\n",
      "intervention\n",
      "bruce\n",
      "draw\n",
      "systems\n",
      "administration\n",
      "ads\n",
      "digital\n",
      "area\n",
      "firm\n",
      "masonbruce\n",
      "graphql\n",
      "survivors\n",
      "united\n",
      "location\n",
      "know\n",
      "support\n",
      "challenging\n",
      "months\n",
      "information\n",
      "big\n",
      "california\n",
      "play\n",
      "disability\n",
      "title\n",
      "class\n",
      "beauty\n",
      "enjoy\n",
      "intern\n",
      "students\n",
      "geog\n",
      "fill\n",
      "hated\n",
      "several\n",
      "usfs\n",
      "dependent\n",
      "open\n",
      "results\n",
      "duties\n",
      "maintain\n",
      "next\n",
      "100\n",
      "stack\n",
      "android\n",
      "abuse\n",
      "expand\n",
      "publishing\n",
      "crisis\n",
      "apply\n",
      "excited\n",
      "www\n",
      "learn\n",
      "francisco\n",
      "provide\n",
      "react\n",
      "headquartered\n",
      "techexcel\n",
      "rep\n",
      "inventory\n",
      "fsr\n",
      "part\n",
      "craftsman\n",
      "call\n",
      "department\n",
      "located\n",
      "securities\n",
      "self\n",
      "back\n",
      "eventually\n",
      "7299u\n",
      "progress\n",
      "perform\n",
      "player\n",
      "skincare\n",
      "fully\n",
      "passionate\n",
      "training\n",
      "computers\n",
      "places\n",
      "databases\n",
      "supportive\n",
      "interes\n",
      "solid\n",
      "behalf\n",
      "berkeley\n",
      "paced\n",
      "com\n",
      "profile\n",
      "environment\n",
      "regular\n",
      "drop\n",
      "package\n",
      "imports\n",
      "public\n",
      "technology\n",
      "institutional\n",
      "enviro\n",
      "dream\n",
      "var\n",
      "dynamic\n",
      "app\n",
      "games\n",
      "optimize\n",
      "user\n",
      "idealistic\n",
      "stores\n",
      "accreditation\n",
      "globally\n",
      "works\n",
      "watch\n",
      "safely\n",
      "remote\n",
      "p_aaaaaacaaadpa\n",
      "program\n",
      "name\n",
      "first\n",
      "link\n",
      "extractions\n",
      "aws\n",
      "vehicles\n",
      "best\n",
      "temporary\n",
      "insurance\n",
      "autonomous\n",
      "developing\n",
      "quickbooks\n",
      "award\n",
      "hire\n",
      "course\n",
      "resource\n",
      "report\n",
      "coverage\n",
      "source\n",
      "tools\n",
      "violence\n",
      "consultants\n",
      "database\n",
      "updates\n",
      "view\n",
      "financial\n",
      "nav\n",
      "mason\n",
      "large\n",
      "geospatial\n",
      "team\n",
      "agency\n",
      "cto\n",
      "military\n",
      "summary\n",
      "creative\n",
      "get\n",
      "landed\n",
      "loved\n",
      "domestic\n",
      "oversight\n",
      "health\n",
      "organized\n",
      "county\n",
      "enhance\n",
      "dental\n",
      "aperia\n",
      "founder\n",
      "san\n",
      "future\n",
      "term\n",
      "spatial\n",
      "upgrade\n",
      "https\n",
      "qualifications\n",
      "contribute\n",
      "offices\n",
      "debug\n",
      "field\n",
      "investment\n",
      "upload\n",
      "consideration\n",
      "profitability\n",
      "craigslist\n",
      "features\n",
      "free\n",
      "already\n",
      "mssql\n",
      "scientists\n",
      "various\n",
      "individual\n",
      "established\n",
      "girard\n",
      "access\n",
      "plus\n",
      "action\n",
      "companies\n",
      "includes\n",
      "said\n",
      "creates\n",
      "bright\n",
      "erp\n",
      "imaging\n",
      "navigating\n",
      "apps\n",
      "stake\n",
      "person\n",
      "arizona\n",
      "fast\n",
      "locations\n",
      "cora\n",
      "gener\n",
      "mappings\n",
      "cutting\n",
      "fragrance\n",
      "natural\n",
      "coverag\n",
      "overview\n",
      "superior\n",
      "holders\n",
      "users\n",
      "contract\n",
      "land\n",
      "candles\n",
      "constfuct\n",
      "challenges\n",
      "projects\n",
      "site\n",
      "world\n",
      "charts\n",
      "programmer\n",
      "modernizes\n",
      "end\n",
      "going\n",
      "provider\n",
      "interactive\n",
      "bugs\n",
      "winning\n",
      "2014\n",
      "come\n",
      "arcturus\n",
      "efficiency\n",
      "introduction\n",
      "opportunities\n",
      "aperiatechcom\n",
      "hundreds\n",
      "growing\n",
      "existing\n",
      "locate\n",
      "including\n",
      "casino\n",
      "franchise\n",
      "fulfill\n",
      "engineers\n",
      "relational\n",
      "netline\n",
      "comm\n",
      "campus\n",
      "enterprise\n",
      "suggest\n",
      "privately\n",
      "serverless\n",
      "job\n",
      "created\n",
      "improve\n",
      "nvwbswx6\n",
      "conversions\n",
      "offer\n",
      "solutions\n",
      "university\n",
      "virtualization\n",
      "music\n",
      "dynamics\n",
      "oconus\n",
      "javascript\n",
      "kids\n",
      "michigan\n",
      "following\n",
      "motivated\n",
      "shj5bh6vtu8\n",
      "servic\n",
      "mini\n",
      "see\n"
     ]
    }
   ],
   "source": [
    "getTopWords(ny,sf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sklearn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX1 : Spam"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Word Vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction import text\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ham\n",
    "files = \"./Data/CH4/email/ham/*.txt\"\n",
    "ham_lst = glob.glob(files)\n",
    "labels = [0]*len(ham_lst)\n",
    "files = \"./Data/CH4/email/spam/*.txt\"\n",
    "spam_lst = glob.glob(files)\n",
    "lst = ham_lst + spam_lst\n",
    "labels.extend([1]*len(spam_lst))\n",
    "cVct = text.CountVectorizer(input=\"filename\",encoding=\"latin_1\")\n",
    "X = cVct.fit_transform(lst).todense().A\n",
    "y = labels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### BernoulliNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import BernoulliNB\n",
    "from sklearn.model_selection import train_test_split\n",
    "bnb = BernoulliNB()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "error rate is 10.00%\n"
     ]
    }
   ],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.2,random_state=0,stratify=y)\n",
    "bnb.fit(X_train,y_train)\n",
    "y_pred = bnb.predict(X_test)\n",
    "print(\"error rate is %1.2f%%\" % (100*np.mean(y_pred != y_test)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "以上\n",
    "- 生成样本的时候有顺序，训练和测试分割的时候记得stratify，不然错误率会和随机种子变化\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import cross_validate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test score is 0.93\n"
     ]
    }
   ],
   "source": [
    "result = cross_validate(X=X,y=y,cv=10,estimator=bnb)\n",
    "print(\"test score is %1.2f\" % (np.mean(result[\"test_score\"])))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EX2 ：Local Word"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- RawData->WordList\n",
    "- RawData->Vect"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Demo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 467,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "X=\"smirk smirk smirk nice smirk comming comming comming back back is is a g\".split(\" \")\n",
    "cVct = CountVectorizer(input=\"content\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 468,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 1, 0],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 0, 0],\n",
       "       [0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 468,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result = cVct.fit_transform(X)\n",
    "result.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 469,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['back', 'comming', 'is', 'nice', 'smirk']"
      ]
     },
     "execution_count": 469,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cVct.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'back': 0, 'comming': 1, 'is': 2, 'nice': 3, 'smirk': 4}"
      ]
     },
     "execution_count": 470,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cVct.vocabulary_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       [0, 0, 0, 1, 0],\n",
       "       [0, 0, 0, 0, 1],\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [0, 1, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 0],\n",
       "       [1, 0, 0, 0, 0],\n",
       "       [0, 0, 1, 0, 0],\n",
       "       [0, 0, 1, 0, 0],\n",
       "       [0, 0, 0, 0, 0],\n",
       "       [0, 0, 0, 0, 0]], dtype=int64)"
      ]
     },
     "execution_count": 472,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 464,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getMostFreq(cVct,X):\n",
    "    tf = sum(X.toarray()).tolist()\n",
    "    tf = dict(zip(cVct.get_feature_names(),tf))\n",
    "    return sorted(tf.items(),key=lambda d:d[1],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('smirk', 4), ('comming', 3), ('back', 2), ('is', 2), ('nice', 1)]"
      ]
     },
     "execution_count": 473,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "getMostFreq(cVct,result)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### CountVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 454,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import feedparser\n",
    "ny = feedparser.parse(\"https://newyork.craigslist.org/search/sof?format=rss\")\n",
    "sf = feedparser.parse(\"https://sfbay.craigslist.org/search/sof?format=rss\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 484,
   "metadata": {},
   "outputs": [],
   "source": [
    "cVct = CountVectorizer(stop_words=stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 485,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('looking', 11),\n",
       " ('software', 9),\n",
       " ('company', 8),\n",
       " ('work', 8),\n",
       " ('developer', 7),\n",
       " ('design', 6),\n",
       " ('java', 6),\n",
       " ('data', 5),\n",
       " ('applications', 4),\n",
       " ('business', 4),\n",
       " ('com', 4),\n",
       " ('develop', 4),\n",
       " ('engineer', 4),\n",
       " ('experience', 4),\n",
       " ('family', 4),\n",
       " ('qa', 4),\n",
       " ('science', 4),\n",
       " ('senior', 4),\n",
       " ('time', 4),\n",
       " ('year', 4),\n",
       " ('analyst', 3),\n",
       " ('ba', 3),\n",
       " ('based', 3),\n",
       " ('beach', 3),\n",
       " ('candidates', 3),\n",
       " ('cloud', 3),\n",
       " ('code', 3),\n",
       " ('cold', 3),\n",
       " ('computer', 3),\n",
       " ('developers', 3)]"
      ]
     },
     "execution_count": 485,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = [x['summary'] for x in ny['entries']]\n",
    "result = cVct.fit_transform(X)\n",
    "getMostFreq(cVct,result)[:30]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 486,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('company', 8),\n",
       " ('developer', 6),\n",
       " ('experience', 6),\n",
       " ('full', 6),\n",
       " ('games', 6),\n",
       " ('re', 6),\n",
       " ('code', 5),\n",
       " ('craigslist', 5),\n",
       " ('design', 5),\n",
       " ('growing', 5),\n",
       " ('job', 5),\n",
       " ('millions', 5),\n",
       " ('mobile', 5),\n",
       " ('new', 5),\n",
       " ('seeks', 5),\n",
       " ('services', 5),\n",
       " ('software', 5),\n",
       " ('test', 5),\n",
       " ('california', 4),\n",
       " ('contribute', 4),\n",
       " ('daily', 4),\n",
       " ('developers', 4),\n",
       " ('development', 4),\n",
       " ('draw', 4),\n",
       " ('driven', 4),\n",
       " ('enjoy', 4),\n",
       " ('hated', 4),\n",
       " ('loved', 4),\n",
       " ('open', 4),\n",
       " ('projects', 4)]"
      ]
     },
     "execution_count": 486,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = [x['summary'] for x in sf['entries']]\n",
    "result = cVct.fit_transform(X)\n",
    "getMostFreq(cVct,result)[:30]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TFIDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## References\n",
    "\n",
    "- 黑客与画家CH8\n",
    "- Sklearn中有[文本分类的例子](http://scikit-learn.org/stable/auto_examples/text/document_classification_20newsgroups.html#sphx-glr-auto-examples-text-document-classification-20newsgroups-py)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "275px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}