Behavior Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "import datetime "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#sourceCheckins = pd.read_csv('data/July/server/checkins.tsv', sep='\\t')\n",
    "allCheckins = pd.read_csv('data/July/server/currentExtendedCheckinsHistory.tsv', sep='\\t')\n",
    "# checkpoints = pd.read_csv('data/July/server/checkPoints.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3445"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(np.unique(allCheckins.userId.tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'Unnamed: 0', u'categoryId', u'categoryName', u'checkinId',\n",
       "       u'checkinsCount', u'createdAt', u'createdAtDate', u'lat', u'lng',\n",
       "       u'name', u'tipCount', u'userId', u'usersCount', u'venueId'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allCheckins.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "allCheckins = allCheckins.drop_duplicates(subset = ['checkinId'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allCheckins = pd.concat([checkins1, checkins])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allCheckins = pd.read_csv('data/July/server/currentExtendedCheckinsHistory.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 399,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "allCheckins.to_csv('data/July/server/currentExtendedCheckinsHistory.tsv', sep='\\t', encoding='utf-8')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "allCheckins.drop('Unnamed: 0', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "sortedVenues = []\n",
    "sortedVenuesIds = []\n",
    "for venueId, group in allCheckins.groupby('venueId').agg('size').order(ascending = False).iteritems():\n",
    "    sortedVenues.append({'venueId':venueId,\n",
    "                        'group':group})\n",
    "    sortedVenuesIds.append(venueId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#'World Class Lite in top ten!, 520f117111d2a782f1211c3b'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "groupedUsers = allCheckins.groupby('userId')\n",
    "userVenues = []\n",
    "for index, group in enumerate(groupedUsers):\n",
    "        userVenues.append(group[1]['venueId'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "83292"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sortedVenues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "top50VenuesCheckins = allCheckins[allCheckins.venueId.isin(sortedVenuesIds[:50])]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'venueNameFromId' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-17-dc9ade34b1da>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mvenueId\u001b[0m \u001b[0;32min\u001b[0m \u001b[0msortedVenuesIds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mprint\u001b[0m \u001b[0mvenueNameFromId\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvenueId\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'venueNameFromId' is not defined"
     ]
    }
   ],
   "source": [
    "for venueId in sortedVenuesIds[:50]:\n",
    "    print venueNameFromId(venueId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from pymining import itemmining\n",
    "relim_input = itemmining.get_relim_input(userVenues)\n",
    "report = itemmining.relim(relim_input, min_support=2)\n",
    "report"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_dataset():\n",
    "    \"Load the sample dataset.\"\n",
    "    return userVenues\n",
    " \n",
    " \n",
    "def createC1(dataset):\n",
    "    \"Create a list of candidate item sets of size one.\"\n",
    "    c1 = []\n",
    "    for transaction in dataset:\n",
    "        for item in transaction:\n",
    "            if not [item] in c1:\n",
    "                c1.append([item])\n",
    "    c1.sort()\n",
    "    #frozenset because it will be a ket of a dictionary.\n",
    "    return map(frozenset, c1)\n",
    " \n",
    " \n",
    "def scanD(dataset, candidates, min_support):\n",
    "    \"Returns all candidates that meets a minimum support level\"\n",
    "    sscnt = {}\n",
    "    for tid in dataset:\n",
    "        for can in candidates:\n",
    "            if can.issubset(tid):\n",
    "                sscnt.setdefault(can, 0)\n",
    "                sscnt[can] += 1\n",
    " \n",
    "    num_items = float(len(dataset))\n",
    "    retlist = []\n",
    "    support_data = {}\n",
    "    for key in sscnt:\n",
    "        support = sscnt[key] / num_items\n",
    "        if support >= min_support:\n",
    "            retlist.insert(0, key)\n",
    "        support_data[key] = support\n",
    "    return retlist, support_data\n",
    " \n",
    " \n",
    "def aprioriGen(freq_sets, k):\n",
    "    \"Generate the joint transactions from candidate sets\"\n",
    "    retList = []\n",
    "    lenLk = len(freq_sets)\n",
    "    for i in range(lenLk):\n",
    "        for j in range(i + 1, lenLk):\n",
    "            L1 = list(freq_sets[i])[:k - 2]\n",
    "            L2 = list(freq_sets[j])[:k - 2]\n",
    "            L1.sort()\n",
    "            L2.sort()\n",
    "            if L1 == L2:\n",
    "                retList.append(freq_sets[i] | freq_sets[j])\n",
    "    return retList\n",
    " \n",
    " \n",
    "def apriori(dataset, minsupport=0.5):\n",
    "    \"Generate a list of candidate item sets\"\n",
    "    C1 = createC1(dataset)\n",
    "    D = map(set, dataset)\n",
    "    L1, support_data = scanD(D, C1, minsupport)\n",
    "    L = [L1]\n",
    "    k = 2\n",
    "    while (len(L[k - 2]) > 0):\n",
    "        Ck = aprioriGen(L[k - 2], k)\n",
    "        Lk, supK = scanD(D, Ck, minsupport)\n",
    "        support_data.update(supK)\n",
    "        L.append(Lk)\n",
    "        k += 1\n",
    " \n",
    "    return L, support_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "C1 = createC1(userVenues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 252,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "D = map(set, userVenues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 253,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "L1, support_data = scanD(D, C1, 0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "L, supportData = apriori(userVenues, minsupport=0.01)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "frequentVenues = []\n",
    "for itemset, support in zip(L, supportData):\n",
    "    if len(itemset)>5:\n",
    "        #print support\n",
    "        frequentVenueSet = []\n",
    "        for item in itemset:\n",
    "            frequentVenueSet.append(next(enumerate(item))[1])\n",
    "        frequentVenues.append(frequentVenueSet)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0000000000027"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(userVenues)*0.0015455950541"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Афимолл Сити / Afimall City\n",
      "ТРЦ РИО «Ленинский»\n",
      "Гин-но Таки\n",
      "McDonald's\n",
      "Cuba Libre\n",
      "Hard Rock Cafe Moscow\n",
      "Conversation Cafe\n",
      "Bar BQ Cafe\n",
      "Каро Vegas 22\n",
      "Счастье\n",
      "Площадь Восстания\n",
      "Starbucks\n",
      "Аэроэкспресс - Терминал на Павелецком вокзале\n",
      "Osteria della Piazza Bianca\n",
      "ТРK «Вегас» (Крокус Сити)\n",
      "Biblioteka\n",
      "ТК «Горбушка»\n",
      "Солнечногорск\n",
      "Багратион\n",
      "Государственная Третьяковская галерея на Крымском Валу\n"
     ]
    }
   ],
   "source": [
    "for venueId in frequentVenues[0][:20]:\n",
    "    print venueNameFromId(venueId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generateRules(L, support_data, min_confidence=0.7):\n",
    "    \"\"\"Create the association rules\n",
    "    L: list of frequent item sets\n",
    "    support_data: support data for those itemsets\n",
    "    min_confidence: minimum confidence threshold\n",
    "    \"\"\"\n",
    "    rules = []\n",
    "    for i in range(1, len(L)):\n",
    "        for freqSet in L[i]:\n",
    "            H1 = [frozenset([item]) for item in freqSet]\n",
    "            #print \"freqSet\", freqSet, 'H1', H1\n",
    "            if (i > 1):\n",
    "                rules_from_conseq(freqSet, H1, support_data, rules, min_confidence)\n",
    "            else:\n",
    "                calc_confidence(freqSet, H1, support_data, rules, min_confidence)\n",
    "    return rules\n",
    " \n",
    " \n",
    "def calc_confidence(freqSet, H, support_data, rules, min_confidence=0.7):\n",
    "    \"Evaluate the rule generated\"\n",
    "    pruned_H = []\n",
    "    for conseq in H:\n",
    "        conf = support_data[freqSet] / support_data[freqSet - conseq]\n",
    "        if conf >= min_confidence:\n",
    "            print freqSet - conseq, '--->', conseq, 'conf:', conf\n",
    "            rules.append((freqSet - conseq, conseq, conf))\n",
    "            pruned_H.append(conseq)\n",
    "    return pruned_H\n",
    " \n",
    " \n",
    "def rules_from_conseq(freqSet, H, support_data, rules, min_confidence=0.7):\n",
    "    \"Generate a set of candidate rules\"\n",
    "    m = len(H[0])\n",
    "    if (len(freqSet) > (m + 1)):\n",
    "        Hmp1 = aprioriGen(H, m + 1)\n",
    "        Hmp1 = calc_confidence(freqSet, Hmp1,  support_data, rules, min_confidence)\n",
    "        if len(Hmp1) > 1:\n",
    "            rules_from_conseq(freqSet, Hmp1, support_data, rules, min_confidence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "rules = generateRules(L, supportData, min_confidence=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Helper Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "venuesDict = {}\n",
    "def venueNameFromId(venueId):\n",
    "    if venueId not in venuesDict.keys():\n",
    "        venuesDict[venueId] = allCheckins[allCheckins.venueId == venueId].iloc[0]['name']\n",
    "    return venuesDict[venueId]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "venuesCatDict = {}\n",
    "def venueCategoryFromId(venueId):\n",
    "    if venueId not in venuesCatDict:\n",
    "        venuesCatDict[venueId] = allCheckins[allCheckins.venueId == venueId].iloc[0]['categoryName']\n",
    "    return venuesCatDict[venueId]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "venuesCatIdsDict = {}\n",
    "def venueCategoryIdFromName(categoryName):\n",
    "    if categoryName not in venuesCatIdsDict:\n",
    "        venuesCatIdsDict[categoryName] = allCheckins[allCheckins.categoryName == categoryName].iloc[0]['categoryId']\n",
    "    return venuesCatIdsDict[categoryName]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "categoriesDict = {}\n",
    "def categoryNameFromId(categoryId):\n",
    "    if categoryId not in categoriesDict.keys():\n",
    "        categoriesDict[categoryId] = allCheckins[allCheckins.categoryId == categoryId].iloc[0]['categoryName']\n",
    "    return categoriesDict[categoryId]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def countUsersAtVenue(venuedId):\n",
    "    return len(np.unique(allCheckins[allCheckins.venueId == venueId]['userId'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Ленинградский вокзал / Leningradsky Railway Terminal\n",
      "Дворцовая площадь / Palace Square\n",
      "--->\n",
      "Невский проспект / Nevsky Prospect\n",
      "Московский вокзал / Moskovsky Railway Station\n",
      "conf: 0.395833333333\n",
      "\n",
      "Ленинградский вокзал / Leningradsky Railway Terminal\n",
      "Невский проспект / Nevsky Prospect\n",
      "--->\n",
      "Дворцовая площадь / Palace Square\n",
      "Московский вокзал / Moskovsky Railway Station\n",
      "conf: 0.520547945205\n",
      "\n",
      "Дворцовая площадь / Palace Square\n",
      "Московский вокзал / Moskovsky Railway Station\n",
      "--->\n",
      "Ленинградский вокзал / Leningradsky Railway Terminal\n",
      "Невский проспект / Nevsky Prospect\n",
      "conf: 0.351851851852\n",
      "\n",
      "Невский проспект / Nevsky Prospect\n",
      "Московский вокзал / Moskovsky Railway Station\n",
      "--->\n",
      "Ленинградский вокзал / Leningradsky Railway Terminal\n",
      "Дворцовая площадь / Palace Square\n",
      "conf: 0.463414634146\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for rule in rules:\n",
    "    simpleRule = 1\n",
    "    if len(rule[0]) < 2:\n",
    "        continue\n",
    "        \n",
    "    for venueId in enumerate(rule[0]):\n",
    "        if venueId[1] not in sortedVenuesIds[:20]:\n",
    "            simpleRule = 0\n",
    "            \n",
    "    if simpleRule:\n",
    "        continue\n",
    "    \n",
    "    simpleRule = 1\n",
    "    \n",
    "    for venueId in enumerate(rule[1]):\n",
    "        if venueId[1] not in sortedVenuesIds[:20]:\n",
    "            simpleRule = 0\n",
    "            \n",
    "    if simpleRule:\n",
    "        continue\n",
    "    \n",
    "    for venueId in enumerate(rule[0]):\n",
    "        print venueNameFromId(venueId[1])\n",
    "        \n",
    "    print '--->'\n",
    "    for venueId in enumerate(rule[1]):\n",
    "        print venueNameFromId(venueId[1])\n",
    "    print 'conf:', rule[2]\n",
    "    print ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Using code for rules from here: http://aimotion.blogspot.ru/2013/01/machine-learning-and-data-mining.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 268,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "multiUserSortedVenues = filter(lambda x: x['group'] > 1, sortedVenues)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "userVenueGroups = []\n",
    "for venueId, group in allCheckins.groupby('venueId'):\n",
    "    if len(group.groupby('userId'))>10:\n",
    "        userVenueGroups.append(venueId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "userVenueCategories = []\n",
    "for categoryId, group in allCheckins.groupby('categoryId'):\n",
    "    if len(group.groupby('userId'))>10:\n",
    "        userVenueCategories.append(categoryId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Make lists of users check-in venues\n",
    "multiUserCheckins = allCheckins[allCheckins.venueId.isin(userVenueGroups)]\n",
    "groupedUsers = multiUserCheckins.groupby('userId')\n",
    "userVenuesList = []\n",
    "userIds = []\n",
    "for index, group in enumerate(groupedUsers):\n",
    "        userVenuesList.append(group[1]['venueId'].tolist())\n",
    "        userIds.append(group[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Make lists of users check-in catrgories\n",
    "fineCheckins = allCheckins[allCheckins.categoryId.isin(userVenueCategories)]\n",
    "groupedUsers = fineCheckins.groupby('userId')\n",
    "userCategoryList = []\n",
    "userIdsForCat = []\n",
    "for index, group in enumerate(groupedUsers):\n",
    "        userCategoryList.append(group[1]['categoryId'].tolist())\n",
    "        userIdsForCat.append(group[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2544"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(userVenueGroups)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-means, spectral"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#Let's build some TF-ID vectors\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.cluster.bicluster import SpectralCoclustering\n",
    "from sklearn.cluster import MiniBatchKMeans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "venuesByUser = []\n",
    "for userList in userVenuesList:\n",
    "    venuesByUser.append(' '.join(userList))\n",
    "    \n",
    "categoriesByUser = []\n",
    "for userList in userCategoryList:\n",
    "    try:\n",
    "        categoriesByUser.append(' '.join(userList))\n",
    "    except:\n",
    "        print userList\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer(vocabulary = userVenueGroups)\n",
    "tfidf_matrix_train = tfidf_vectorizer.fit_transform(venuesByUser)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Coclustering...\n",
      "Done in 0.57s.\n",
      "MiniBatchKMeans...\n",
      "Done in 0.11s.\n"
     ]
    }
   ],
   "source": [
    "from time import time\n",
    "cocluster = SpectralCoclustering(n_clusters=20,\n",
    "                                 svd_method='arpack', random_state=0)\n",
    "kmeans = MiniBatchKMeans(n_clusters=20,\n",
    "                         random_state=0)\n",
    "\n",
    "print(\"Coclustering...\")\n",
    "start_time = time()\n",
    "cocluster.fit(tfidf_matrix_train)\n",
    "y_cocluster = cocluster.row_labels_\n",
    "print(\"Done in {:.2f}s.\".format(\n",
    "    time() - start_time))\n",
    "\n",
    "print(\"MiniBatchKMeans...\")\n",
    "start_time = time()\n",
    "y_kmeans = kmeans.fit_predict(tfidf_matrix_train)\n",
    "print(\"Done in {:.2f}s.\".format(\n",
    "    time() - start_time))\n",
    "\n",
    "feature_names = tfidf_vectorizer.get_feature_names()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clustersSpectral = [[] for i in range(20)]\n",
    "for index, clusterId in enumerate(y_cocluster):\n",
    "    clustersSpectral[clusterId].append(userIds[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'clusters' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-42-5651f0b9a819>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcluster\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mcluster\u001b[0m \u001b[0;32min\u001b[0m  \u001b[0mclusters\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'clusters' is not defined"
     ]
    }
   ],
   "source": [
    "np.std([len(cluster) for cluster in  clusters])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clusters = x = [[] for i in range(20)]\n",
    "for index, clustedId in enumerate(y_kmeans):\n",
    "    clusters[clustedId].append(userIds[index])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LDA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Document matrix\n"
     ]
    }
   ],
   "source": [
    "import sklearn\n",
    "from sklearn.feature_extraction.text import CountVectorizer \n",
    "\n",
    "cv = sklearn.feature_extraction.text.CountVectorizer()\n",
    "mat = cv.fit_transform(venuesByUser).toarray()\n",
    "print('Done Document matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "135164"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum([len(venuesUser) for venuesUser in userVenuesList])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done Document matrix\n"
     ]
    }
   ],
   "source": [
    "cvCategory = sklearn.feature_extraction.text.CountVectorizer()\n",
    "matCategory = cvCategory.fit_transform(categoriesByUser).toarray()\n",
    "print('Done Document matrix')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2544\n"
     ]
    }
   ],
   "source": [
    "venuesVocab = cv.get_feature_names()\n",
    "print len(venuesVocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "461\n"
     ]
    }
   ],
   "source": [
    "categoriesVocab = cvCategory.get_feature_names()\n",
    "print len(categoriesVocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done LDA fitting\n"
     ]
    }
   ],
   "source": [
    "import lda\n",
    "model20 = lda.LDA(n_topics=20, n_iter=2000, random_state=1,alpha = 0.001)\n",
    "model20.fit(mat)  # model.fit_transform(X) is also available\n",
    "topic_word = model20.topic_word_  # model.components_ also works\n",
    "print('Done LDA fitting')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done LDA fitting\n"
     ]
    }
   ],
   "source": [
    "import lda\n",
    "model20Cat = lda.LDA(n_topics=20, n_iter=2000, random_state=1,alpha = 0.001)\n",
    "model20Cat.fit(matCategory)  # model.fit_transform(X) is also available\n",
    "topic_cats = model20Cat.topic_word_  # model.components_ also works\n",
    "print('Done LDA fitting')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 1:\n",
      "ВДНХ (Выставка достижений народного хозяйства), 994 users 4.58%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 2.90%\n",
      "ТРЦ «Золотой Вавилон», 208 users 2.87%\n",
      "Ярославский вокзал, 193 users 2.22%\n",
      "Красная площадь / Red Square, 755 users 2.15%\n",
      "\n",
      "Cluster 2:\n",
      "Физика, 44 users 6.81%\n",
      "Афимолл Сити / Afimall City, 455 users 6.30%\n",
      "ТРЦ «Европейский», 539 users 4.80%\n",
      "ТРЦ «Филион», 110 users 3.60%\n",
      "ТЦ «Кунцево Плаза», 66 users 2.81%\n",
      "ПКиО «Фили», 144 users 2.59%\n",
      "Парк Победы (Поклонная гора) / Victory Park, 366 users 2.42%\n",
      "Арбат / Arbat Street, 557 users 2.21%\n",
      "ALEX-Fitness, 24 users 1.99%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 1.51%\n",
      "ТЦ «ЕвроПарк» / EuroPark Mall, 59 users 1.39%\n",
      "Башня «Федерация» / Federation Tower, 49 users 1.32%\n",
      "Район «Крылатское», 23 users 1.01%\n",
      "\n",
      "Cluster 3:\n",
      "Москва / Moscow, 656 users 38.49%\n",
      "Россия / Russia, 321 users 18.77%\n",
      "Западный административный округ, 148 users 3.11%\n",
      "Северный административный округ, 95 users 2.26%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 1.98%\n",
      "Красная площадь / Red Square, 755 users 1.77%\n",
      "Парк искусств «Музеон» / Muzeon Park, 975 users 1.52%\n",
      "\n",
      "Cluster 4:\n",
      "Московский вокзал / Moskovsky Railway Station, 311 users 6.23%\n",
      "Ленинградский вокзал / Leningradsky Railway Terminal, 431 users 5.29%\n",
      "Международный аэропорт Пулково / Pulkovo International Airport (LED), 178 users 3.14%\n",
      "Невский проспект / Nevsky Prospect, 147 users 2.93%\n",
      "ТРЦ «Галерея» / Galeria Shopping Mall, 133 users 2.67%\n",
      "Дворцовая площадь / Palace Square, 172 users 2.56%\n",
      "Санкт-Петербург, 74 users 1.44%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 1.34%\n",
      "Государственный Эрмитаж / Hermitage Museum, 93 users 1.12%\n",
      "\n",
      "Cluster 5:\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 5.22%\n",
      "ТРЦ «Columbus», 141 users 3.52%\n",
      "Alex Fitness, 23 users 2.70%\n",
      "ТРК «Капитолий», 149 users 2.07%\n",
      "МЕГА Тёплый Стан / MEGA Mall, 144 users 1.63%\n",
      "МГЛУ / Московский государственный лингвистический университет / Moscow State Linguistic University, 23 users 1.49%\n",
      "РГУ нефти и газа имени И. М. Губкина, 20 users 1.42%\n",
      "Парк искусств «Музеон» / Muzeon Park, 975 users 1.33%\n",
      "Нескучный сад, 277 users 1.07%\n",
      "Воробьёвская набережная, 264 users 1.04%\n",
      "Арбитражный суд Москвы, 28 users 1.03%\n",
      "Музей-заповедник «Коломенское», 374 users 1.03%\n",
      "Битцевский лесопарк, 66 users 1.00%\n",
      "\n",
      "Cluster 6:\n",
      "РЭУ им. Г. В. Плеханова, 55 users 12.24%\n",
      "Факультет журналистики МГУ, 28 users 7.30%\n",
      "Бассейн «Олимпийский», 29 users 6.55%\n",
      "Зебра, 11 users 5.59%\n",
      "Бизнес-парк «Крылатские холмы», 11 users 3.17%\n",
      "ТРЦ «Райкин Плаза», 73 users 2.47%\n",
      "Good Enough, 27 users 2.27%\n",
      "Район «Марьина Роща», 28 users 1.91%\n",
      "Сад им. Баумана, 120 users 1.86%\n",
      "Starbucks, 26 users 1.71%\n",
      "Уфа, 13 users 1.51%\n",
      "KFC, 15 users 1.36%\n",
      "ТРК «Капитолий», 46 users 1.36%\n",
      "Шоколадница, 13 users 1.31%\n",
      "Starbucks, 73 users 1.31%\n",
      "ВДНХ (Выставка достижений народного хозяйства), 994 users 1.31%\n",
      "Красная площадь / Red Square, 755 users 1.21%\n",
      "Пушкинская площадь, 219 users 1.21%\n",
      "Библиотека им. Ф. М. Достоевского, 16 users 1.21%\n",
      "Соловей, 170 users 1.11%\n",
      "ТЦ «Охотный ряд», 415 users 1.06%\n",
      "Центральный детский магазин, 531 users 1.06%\n",
      "БЦ «Белая площадь», 30 users 1.01%\n",
      "\n",
      "Cluster 7:\n",
      "Фитнес-мания, 28 users 12.29%\n",
      "Планета Фитнес, 22 users 10.91%\n",
      "Gold's Gym, 15 users 9.33%\n",
      "Телецентр «Останкино», АСК-3, 12 users 3.36%\n",
      "Телецентр «Останкино», АСК-1, 37 users 3.16%\n",
      "Природный заказник «Долина реки Сетунь», 12 users 3.11%\n",
      "ТРK «Вегас» / Vegas Mall, 135 users 2.62%\n",
      "Оки Доки, 20 users 2.57%\n",
      "La Bottega Siciliana, 11 users 2.02%\n",
      "ДжонДжоли, 15 users 1.78%\n",
      "Кампус, 20 users 1.73%\n",
      "Ray Just Arena, 61 users 1.68%\n",
      "Чайхона № 1, 20 users 1.38%\n",
      "ГлавКино, 11 users 1.33%\n",
      "9 залов, 11 users 1.18%\n",
      "ТРК «Торговый квартал», 11 users 1.04%\n",
      "\n",
      "Cluster 8:\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 2.95%\n",
      "Арбат / Arbat Street, 557 users 2.33%\n",
      "Пушкинская площадь, 219 users 2.19%\n",
      "World Gym, 13 users 2.00%\n",
      "Alex Fitness, 13 users 1.90%\n",
      "Тверской бульвар, 89 users 1.90%\n",
      "Парк искусств «Музеон» / Muzeon Park, 975 users 1.83%\n",
      "ТГ «Модный сезон», 49 users 1.66%\n",
      "Район «Строгино», 43 users 1.52%\n",
      "Нескучный сад, 277 users 1.43%\n",
      "ТРК «Щука», 84 users 1.40%\n",
      "Улица Кузнецкий Мост, 94 users 1.33%\n",
      "Podium Market, 31 users 1.31%\n",
      "Центральный детский магазин, 531 users 1.07%\n",
      "Красная площадь / Red Square, 755 users 1.07%\n",
      "ТЦ «Метрополис», 391 users 1.07%\n",
      "ТРЦ «Калейдоскоп», 118 users 1.02%\n",
      "Тверская улица, 114 users 1.02%\n",
      "Новый Арбат / New Arbat Street, 154 users 1.02%\n",
      "ТРЦ «Пятая Авеню», 25 users 1.02%\n",
      "\n",
      "Cluster 9:\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 5.54%\n",
      "Парк искусств «Музеон» / Muzeon Park, 975 users 2.31%\n",
      "ВДНХ (Выставка достижений народного хозяйства), 994 users 1.72%\n",
      "Центральный детский магазин, 531 users 1.25%\n",
      "ТРК «Атриум» / Atrium Mall, 520 users 1.19%\n",
      "Красная площадь / Red Square, 755 users 1.19%\n",
      "Арбат / Arbat Street, 557 users 1.15%\n",
      "\n",
      "Cluster 10:\n",
      "Международный аэропорт Шереметьево / Sheremetyevo International Airport (SVO), 544 users 7.02%\n",
      "Международный аэропорт Домодедово / Domodedovo International Airport (DME), 799 users 6.84%\n",
      "Международный аэропорт Внуково / Vnukovo International Airport (VKO), 526 users 3.46%\n",
      "Терминал D / Terminal D, 192 users 1.98%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 1.24%\n",
      "ПКиО «Сокольники», 637 users 1.07%\n",
      "\n",
      "Cluster 11:\n",
      "ТРЦ «Калейдоскоп», 118 users 4.81%\n",
      "Парк Дружбы, 102 users 4.65%\n",
      "МЕГА Химки / MEGA Mall, 158 users 3.99%\n",
      "Парк «Северное Тушино», 107 users 3.85%\n",
      "World Class Lite, 17 users 3.80%\n",
      "ТЦ «Метрополис», 391 users 3.41%\n",
      "ТЦ «Водный», 67 users 2.03%\n",
      "ТРK «Вегас» (Крокус Сити), 165 users 1.96%\n",
      "ТЦ «Капитолий», 61 users 1.80%\n",
      "IKEA, 147 users 1.73%\n",
      "ТЦ «Речной», 56 users 1.64%\n",
      "Ландшафтный парк Митино, 33 users 1.45%\n",
      "Балтика, 46 users 1.36%\n",
      "МИТРО, 11 users 1.36%\n",
      "Митино, 32 users 1.36%\n",
      "Киносфера, 40 users 1.31%\n",
      "Покровское-Стрешнево (природно-исторический парк), 85 users 1.06%\n",
      "\n",
      "Cluster 12:\n",
      "FitFashion, 14 users 5.88%\n",
      "World Gym Синица, 20 users 5.41%\n",
      "MANXЭТТЕН, 15 users 5.37%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 3.65%\n",
      "Российский университет дружбы народов (РУДН), 48 users 3.61%\n",
      "ВДНХ (Выставка достижений народного хозяйства), 994 users 3.02%\n",
      "Международный аэропорт Внуково / Vnukovo International Airport (VKO), 526 users 2.31%\n",
      "Craft rePUBlic, 38 users 2.04%\n",
      "Все твои друзья, 60 users 2.00%\n",
      "McDonald's, 34 users 1.69%\n",
      "World Class, 11 users 1.61%\n",
      "Pivbar, 59 users 1.49%\n",
      "Гончаровский парк, 13 users 1.49%\n",
      "İstanbul Atatürk Airport (IST), 32 users 1.14%\n",
      "Район «Хамовники», 29 users 1.14%\n",
      "Красная площадь / Red Square, 755 users 1.14%\n",
      "Екатерининский парк, 159 users 1.10%\n",
      "\n",
      "Cluster 13:\n",
      "ТРЦ «Авиапарк», 240 users 4.42%\n",
      "ТЦ «Метрополис», 391 users 3.47%\n",
      "РГГУ (Российский государственный гуманитарный университет), 41 users 2.57%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 1.90%\n",
      "ПКиО «Красная Пресня», 168 users 1.40%\n",
      "Петровский парк, 43 users 1.24%\n",
      "Дом Правительства РФ, 23 users 1.22%\n",
      "ТРЦ «Европейский», 539 users 1.07%\n",
      "Парк «Дубки», 24 users 1.07%\n",
      "\n",
      "Cluster 14:\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 3.35%\n",
      "World Class Lite, 15 users 1.87%\n",
      "GIPSY, 163 users 1.72%\n",
      "Международный аэропорт Домодедово / Domodedovo International Airport (DME), 799 users 1.48%\n",
      "Патриаршие пруды / Patriarshiye Ponds, 399 users 1.35%\n",
      "Международный аэропорт Шереметьево / Sheremetyevo International Airport (SVO), 544 users 1.07%\n",
      "Афимолл Сити / Afimall City, 455 users 1.03%\n",
      "\n",
      "Cluster 15:\n",
      "ПКиО «Сокольники», 637 users 10.64%\n",
      "Яндекс / Yandex HQ, 51 users 6.87%\n",
      "МГУДТ (Московский государственный университет дизайна и технологий), 14 users 3.50%\n",
      "Курский вокзал / Kursky Rail Terminal, 288 users 3.19%\n",
      "Изя гриль, 33 users 3.06%\n",
      "Район «Сокольники», 72 users 2.60%\n",
      "Стадион «Торпедо» им. Э.А. Стрельцова, 20 users 2.47%\n",
      "Станция Железнодорожная, 12 users 2.23%\n",
      "Свиблово, 29 users 2.04%\n",
      "Железнодорожный, 19 users 1.70%\n",
      "Сиреневый сад, 69 users 1.64%\n",
      "ТРК «Атриум» / Atrium Mall, 520 users 1.36%\n",
      "Кофе Хауз, 23 users 1.21%\n",
      "Зебра, 11 users 1.18%\n",
      "Ашан, 47 users 1.14%\n",
      "McDonald's, 54 users 1.11%\n",
      "\n",
      "Cluster 16:\n",
      "Музей-заповедник «Коломенское», 374 users 3.00%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 2.11%\n",
      "ПКиО «Сокольники», 637 users 1.87%\n",
      "ВДНХ (Выставка достижений народного хозяйства), 994 users 1.63%\n",
      "Музей-заповедник «Царицыно», 333 users 1.56%\n",
      "ТРК «Атриум» / Atrium Mall, 520 users 1.49%\n",
      "ПКиО «Измайловский», 172 users 1.37%\n",
      "Красная площадь / Red Square, 755 users 1.32%\n",
      "ТРЦ «Европейский», 539 users 1.28%\n",
      "МЕГА Белая Дача / MEGA Mall, 234 users 1.23%\n",
      "Арбат / Arbat Street, 557 users 1.17%\n",
      "Международный аэропорт Домодедово / Domodedovo International Airport (DME), 799 users 1.15%\n",
      "ТЦ «Охотный ряд», 415 users 1.14%\n",
      "Москва / Moscow, 656 users 1.10%\n",
      "ТК «Город», 116 users 1.04%\n",
      "\n",
      "Cluster 17:\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 5.17%\n",
      "Парк искусств «Музеон» / Muzeon Park, 975 users 3.93%\n",
      "Tsvetnoy Central Market, 261 users 1.83%\n",
      "Пропаганда, 147 users 1.51%\n",
      "Бар «Стрелка», 150 users 1.19%\n",
      "Сад «Эрмитаж» / Hermitage Garden, 243 users 1.08%\n",
      "\n",
      "Cluster 18:\n",
      "РАНХиГС при Президенте РФ, 55 users 6.89%\n",
      "МГИМО, 35 users 4.97%\n",
      "Фитнес клуб \"Зебра\"/ ZEBRA FITNES and SPA, 13 users 3.69%\n",
      "Метро Бабушкинская (metro Babushkinskaya), 18 users 2.90%\n",
      "World Class, 11 users 2.64%\n",
      "Район «Куркино», 20 users 2.37%\n",
      "Метро Университет (metro Universitet), 22 users 2.17%\n",
      "МГТУ МИРЭА, 11 users 2.17%\n",
      "Метро Водный Стадион (metro Vodniy Stadion), 11 users 1.88%\n",
      "Воробьёвская набережная, 264 users 1.75%\n",
      "ВымпелКом HQ, 13 users 1.68%\n",
      "ТРЦ РИО «Ленинский», 65 users 1.61%\n",
      "Starbucks, 38 users 1.45%\n",
      "Район «Ясенево», 20 users 1.45%\n",
      "Воронцовский парк, 75 users 1.45%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 1.38%\n",
      "ТЦ «Звёздочка», 49 users 1.38%\n",
      "ТРЦ «Европейский», 539 users 1.38%\n",
      "ТРК «Капитолий», 149 users 1.35%\n",
      "McDonald's, 37 users 1.29%\n",
      "Метро Академическая (metro Akademicheskaya), 13 users 1.29%\n",
      "Метро Юго-Западная (metro Yugo-Zapadnaya), 23 users 1.29%\n",
      "МГУ им. М. В. Ломоносова, 103 users 1.25%\n",
      "Петровский парк, 43 users 1.19%\n",
      "Парк искусств «Музеон» / Muzeon Park, 975 users 1.02%\n",
      "\n",
      "Cluster 19:\n",
      "МЕГА Белая Дача / MEGA Mall, 234 users 5.42%\n",
      "Государственный университет управления (ГУУ), 24 users 5.42%\n",
      "СОК «Чайка», 12 users 3.36%\n",
      "Природно-исторический парк «Кузьминки-Люблино», 114 users 2.90%\n",
      "World Gym, 11 users 2.68%\n",
      "ЦПКиО им. Горького / Gorky Park, 1728 users 2.13%\n",
      "ПКиО «Измайловский», 172 users 1.94%\n",
      "ТК «Город», 116 users 1.91%\n",
      "The Сад, 14 users 1.88%\n",
      "World Gym, 14 users 1.88%\n",
      "PPL. People for People, 31 users 1.63%\n",
      "Лианозово, 12 users 1.45%\n",
      "ТРК «Атриум» / Atrium Mall, 520 users 1.42%\n",
      "ТК «Город», 42 users 1.39%\n",
      "Outlet Village Белая Дача, 100 users 1.36%\n",
      "ТРЦ «РИО», 34 users 1.36%\n",
      "Реутов, 18 users 1.20%\n",
      "Балашиха, 34 users 1.17%\n",
      "Район «Новогиреево», 26 users 1.17%\n",
      "Казанский вокзал / Kazansky Rail Terminal, 271 users 1.11%\n",
      "ВДНХ (Выставка достижений народного хозяйства), 994 users 1.11%\n",
      "IKEA, 91 users 1.02%\n",
      "IMAX KinoStar New York, 47 users 1.02%\n",
      "\n",
      "Cluster 20:\n",
      "Московский государственный юридический университет им. О. Е. Кутафина (МГЮА), 17 users 3.52%\n",
      "Волконский, 30 users 2.80%\n",
      "Савеловский вокзал / Savyolovsky Rail Terminal, 64 users 2.53%\n",
      "Метро Савёловская (metro Savyolovskaya), 16 users 2.44%\n",
      "Станция «Одинцово», 23 users 2.15%\n",
      "Белорусский вокзал / Belorussky Rail Terminal, 197 users 2.09%\n",
      "МИА «Россия сегодня», 16 users 1.70%\n",
      "Планета Фитнес, 11 users 1.61%\n",
      "Пресненский район, 19 users 1.55%\n",
      "Улица Красная Пресня, 26 users 1.52%\n",
      "ПКиО «Красная Пресня», 168 users 1.49%\n",
      "ПКиО «Измайловский», 172 users 1.46%\n",
      "Метро Октябрьское поле (metro Oktyabrskoye Pole), 14 users 1.40%\n",
      "Люблинский пруд, 18 users 1.31%\n",
      "Патриаршие пруды / Patriarshiye Ponds, 399 users 1.25%\n",
      "Район «Марьино», 25 users 1.16%\n",
      "Парк Декабрьского восстания, 19 users 1.01%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_top_venues = 10\n",
    "for i, venues_dist in enumerate(topic_word):\n",
    "    indicies = np.argsort(venues_dist)[::-1]\n",
    "    topic_venues = np.array(venuesVocab)[indicies]\n",
    "    topic_probs = venues_dist[indicies]\n",
    "    #print np.sort(venues_dist)[:-n_top_venues:-1]\n",
    "    print('Cluster {}:'.format(i+1))\n",
    "    for venueId, topicProb in zip(topic_venues, topic_probs):\n",
    "        if topicProb>=0.01:\n",
    "            print(\"{}, {} users {:.2f}%\").format(venueNameFromId(venueId),countUsersAtVenue(venueId),topicProb*100)\n",
    "        else:\n",
    "            break\n",
    "    print ''"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Create LDA users clusters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def exportLDAClustersToCSV(venues_clusters, allCheckins):\n",
    "    topVenuesLDA = []\n",
    "    LDAclustersDF = pd.DataFrame()\n",
    "    for clustedId, venues_dist in enumerate(venues_clusters):\n",
    "        indicies = np.argsort(venues_dist)[::-1]\n",
    "        \n",
    "        topic_venues = np.array(venuesVocab)[indicies]\n",
    "        topic_probs = venues_dist[indicies]\n",
    "\n",
    "        topVenuesIds =  [venueId for venueId, topicProb in zip(topic_venues, topic_probs) if topicProb>=0.01]\n",
    "        topVenuesLDA.extend(topVenuesIds)\n",
    "        clusterCheckins = allCheckins[allCheckins.venueId.isin(topVenuesIds)]\n",
    "        clusterCheckins[\"clusterId\"] = clustedId+1\n",
    "        LDAclustersDF = LDAclustersDF.append(clusterCheckins)\n",
    "    print len(topVenuesLDA)\n",
    "    print len(set(topVenuesLDA))\n",
    "    return LDAclustersDF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "271\n",
      "206\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/IPython/kernel/__main__.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "LDAclustersDF = exportLDAClustersToCSV(topic_word, allCheckins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "LDAclustersDF.to_csv('data/July/LDATopVenuesMoreThan10Whole.tsv', sep=\"\\t\", encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "250\n",
      "149\n",
      "114\n",
      "218\n",
      "235\n",
      "38\n",
      "41\n",
      "78\n",
      "620\n",
      "333\n",
      "101\n",
      "87\n",
      "138\n",
      "263\n",
      "61\n",
      "270\n",
      "264\n",
      "51\n",
      "92\n",
      "42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python2.7/site-packages/IPython/kernel/__main__.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "LDAclusteredUsersDF = pd.DataFrame()\n",
    "for clusterId, clusterLDA in enumerate(clustersLDA):\n",
    "    clusterCheckins = allCheckins[allCheckins.userId.isin(clusterLDA)]\n",
    "    clusterCheckins[\"clusterId\"] = clusterId + 1\n",
    "    LDAclusteredUsersDF = LDAclusteredUsersDF.append(clusterCheckins)\n",
    "    print len(clusterLDA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "LDAclusteredUsersDF.to_csv('data/July/LDAClusteredUsersMoreThan10Whole.tsv', sep=\"\\t\", encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "def extractComponents(row):\n",
    "    date = datetime.utcfromtimestamp(row.createdAt+180*60)\n",
    "    hours.append(date.hour)\n",
    "    weekdays.append(date.weekday())\n",
    "    \n",
    "hours = []\n",
    "weekdays = []\n",
    "\n",
    "LDAclusteredUsersDF.apply(extractComponents, axis=1)\n",
    "LDAclusteredUsersDF['hour'] = hours\n",
    "LDAclusteredUsersDF['weekday'] = weekdays"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 476,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " Checkins per user: "
     ]
    },
    {
     "ename": "IndexingError",
     "evalue": "Unalignable boolean Series key provided",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mIndexingError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-476-542970821dd1>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgroup\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mLDAclusteredUsersDF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"clusterId\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0;32mprint\u001b[0m \u001b[0;34m'Checkins per user: '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"createdAtDate\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m6\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m7\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'6'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgroup\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgroupby\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"userId\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1789\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndarray\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mIndex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1790\u001b[0m             \u001b[0;31m# either boolean or fancy integer index\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1791\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_array\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1792\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1793\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_frame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_getitem_array\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   1829\u001b[0m             \u001b[0;31m# check_bool_indexer will throw exception if Series key cannot\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1830\u001b[0m             \u001b[0;31m# be reindexed to match DataFrame rows\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1831\u001b[0;31m             \u001b[0mkey\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcheck_bool_indexer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1832\u001b[0m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnonzero\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1833\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtake\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mconvert\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/local/lib/python2.7/site-packages/pandas/core/indexing.pyc\u001b[0m in \u001b[0;36mcheck_bool_indexer\u001b[0;34m(ax, key)\u001b[0m\n\u001b[1;32m   1657\u001b[0m         \u001b[0mmask\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcom\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misnull\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1658\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1659\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mIndexingError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Unalignable boolean Series key provided'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1660\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1661\u001b[0m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbool\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mIndexingError\u001b[0m: Unalignable boolean Series key provided"
     ]
    }
   ],
   "source": [
    "for index, group in enumerate(LDAclusteredUsersDF.groupby(\"clusterId\")):\n",
    "    print 'Checkins per user: ', len(group[1][group[1][\"createdAtDate\"][6:7] == '6'])/len(group[1].groupby(\"userId\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'clustersLDACategory' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-45-2474a22df0df>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mfor\u001b[0m \u001b[0mclusterLDA\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mclusterLDACat\u001b[0m  \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclustersLDA\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mclustersLDACategory\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m     \u001b[0;32mprint\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclusterLDA\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mclusterLDACat\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'clustersLDACategory' is not defined"
     ]
    }
   ],
   "source": [
    "for clusterLDA,clusterLDACat  in zip(clustersLDA,clustersLDACategory):\n",
    "    print len(clusterLDA), len(clusterLDACat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date = datetime.now()\n",
    "date.weekday()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "134866\n"
     ]
    }
   ],
   "source": [
    "LDACheckins = allCheckins[allCheckins.venueId.isin(venuesVocab)]\n",
    "print len(LDACheckins)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "LDAclusteredCategoriesDF = pd.DataFrame()\n",
    "for clusterId, clusterLDA in enumerate(clustersLDACategory):\n",
    "    clusterCheckins = LDACheckins[LDACheckins.userId.isin(clusterLDA)]\n",
    "    clusterCheckins[\"clusterId\"] = clusterId + 1\n",
    "    LDAclusteredCategoriesDF = LDAclusteredCategoriesDF.append(clusterCheckins)\n",
    "    print len(clusterCheckins)\n",
    "LDAclusteredCategoriesDF.to_csv('data/July/LDAClusteredUsersByCategory.tsv', sep=\"\\t\", encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "clustersLDA = [[] for i in range(20)]\n",
    "for index, userTopics in enumerate(model20.ndz_):\n",
    "    clustersLDA[np.argmax(userTopics)].append(userIds[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clustersLDACategory = [[] for i in range(20)]\n",
    "for index, userTopics in enumerate(model20Cat.ndz_):\n",
    "    clustersLDACategory[np.argmax(userTopics)].append(userIdsForCat[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Match the clusters by min Jaccard Distance\n",
    "import distance\n",
    "LDAClustersMap = []\n",
    "LDAClustersMapIndexes = []\n",
    "# clustersLDACopy = clustersLDA\n",
    "distancesMatrix = []\n",
    "for clusterK in clusters:\n",
    "    distances = [distance.jaccard(clusterLDA,clusterK) for clusterLDA in clustersLDA]\n",
    "    distancesMatrix.append(distances)\n",
    "    index = np.argmin(distances)\n",
    "    LDAClustersMap.append(clustersLDA[index])\n",
    "#     clustersLDACopy.remove(clustersLDA[index])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "clustersSpectral1 = [[0] if len(cluster) == 0 else cluster for cluster in clusterSpectreal]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "distancesMatrixKSpect = []\n",
    "for clusterK in clusters:\n",
    "    distances = [distance.jaccard(clusterSpectreal,clusterK) for clusterSpectreal in clustersSpectral]\n",
    "    distancesMatrixKSpect.append(distances)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.891365675713\n"
     ]
    }
   ],
   "source": [
    "#Average Jaccard Distance between clusters\n",
    "print np.mean([distance.jaccard(clusterK, LDAClustersMap[index]) for index, clusterK in enumerate(clusters) if len(clusterK)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.923955972999\n"
     ]
    }
   ],
   "source": [
    "#Average Jaccard Distance between clusters\n",
    "print np.mean(np.min(distancesMatrix, axis=1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 1:\n",
      "Residential Building (Apartment / Condo), 10.83%\n",
      "Mall, 10.52%\n",
      "Coffee Shop, 9.17%\n",
      "Bank, 6.00%\n",
      "Park, 5.05%\n",
      "Fast Food Restaurant, 4.54%\n",
      "Grocery Store, 2.16%\n",
      "Big Box Store, 2.15%\n",
      "Sushi Restaurant, 1.95%\n",
      "\n",
      "Cluster 2:\n",
      "Park, 5.66%\n",
      "Coffee Shop, 5.09%\n",
      "Airport, 4.82%\n",
      "Mall, 4.81%\n",
      "Hotel, 3.55%\n",
      "Café, 3.08%\n",
      "Train Station, 2.73%\n",
      "City, 2.12%\n",
      "Restaurant, 1.76%\n",
      "\n",
      "Cluster 3:\n",
      "Office, 19.11%\n",
      "Gym / Fitness Center, 17.62%\n",
      "Tech Startup, 6.05%\n",
      "Airport, 3.92%\n",
      "Cocktail Bar, 2.27%\n",
      "Park, 1.97%\n",
      "Medical Center, 1.97%\n",
      "Mall, 1.88%\n",
      "Hotel, 1.83%\n",
      "\n",
      "Cluster 4:\n",
      "Home (private), 43.82%\n",
      "Office, 4.58%\n",
      "Mall, 4.33%\n",
      "Residential Building (Apartment / Condo), 3.73%\n",
      "Gym Pool, 2.25%\n",
      "Capitol Building, 1.96%\n",
      "Financial or Legal Service, 1.37%\n",
      "Airport, 1.33%\n",
      "Clothing Store, 1.29%\n",
      "\n",
      "Cluster 5:\n",
      "Gym / Fitness Center, 66.53%\n",
      "Mall, 6.70%\n",
      "College Communications Building, 1.96%\n",
      "Train Station, 1.67%\n",
      "Park, 1.56%\n",
      "Salon / Barbershop, 1.56%\n",
      "Multiplex, 1.54%\n",
      "Hockey Arena, 1.11%\n",
      "Sushi Restaurant, 1.06%\n",
      "\n",
      "Cluster 6:\n",
      "Park, 16.31%\n",
      "Mall, 5.01%\n",
      "Street, 4.63%\n",
      "Plaza, 3.31%\n",
      "Train Station, 3.25%\n",
      "Coffee Shop, 2.86%\n",
      "Café, 1.94%\n",
      "Lake, 1.90%\n",
      "Neighborhood, 1.86%\n",
      "\n",
      "Cluster 7:\n",
      "Café, 5.63%\n",
      "Airport, 5.13%\n",
      "Park, 4.91%\n",
      "Coffee Shop, 4.34%\n",
      "Nightclub, 3.84%\n",
      "Restaurant, 3.77%\n",
      "Italian Restaurant, 2.91%\n",
      "Bar, 2.90%\n",
      "Cocktail Bar, 2.43%\n",
      "\n",
      "Cluster 8:\n",
      "Home (private), 18.31%\n",
      "Park, 9.04%\n",
      "Street, 8.84%\n",
      "Mall, 6.88%\n",
      "School, 3.54%\n",
      "Neighborhood, 3.42%\n",
      "Plaza, 3.09%\n",
      "Residential Building (Apartment / Condo), 2.26%\n",
      "Fast Food Restaurant, 2.09%\n",
      "\n",
      "Cluster 9:\n",
      "Mall, 14.06%\n",
      "Street, 8.69%\n",
      "Neighborhood, 8.06%\n",
      "Government Building, 7.14%\n",
      "Park, 6.27%\n",
      "Plaza, 3.07%\n",
      "Gas Station / Garage, 2.60%\n",
      "Parking, 2.59%\n",
      "Fast Food Restaurant, 2.04%\n",
      "\n",
      "Cluster 10:\n",
      "University, 15.29%\n",
      "Mall, 10.01%\n",
      "Park, 5.66%\n",
      "College Academic Building, 4.06%\n",
      "Dance Studio, 3.67%\n",
      "Coffee Shop, 3.54%\n",
      "School, 3.52%\n",
      "College Arts Building, 2.88%\n",
      "Hookah Bar, 2.39%\n",
      "\n",
      "Cluster 11:\n",
      "Home (private), 66.68%\n",
      "Park, 2.86%\n",
      "Hospital, 2.58%\n",
      "Neighborhood, 2.27%\n",
      "City, 2.09%\n",
      "Advertising Agency, 1.75%\n",
      "Dentist's Office, 1.63%\n",
      "Plaza, 1.60%\n",
      "Train Station, 1.28%\n",
      "\n",
      "Cluster 12:\n",
      "Residential Building (Apartment / Condo), 23.33%\n",
      "Metro Station, 22.33%\n",
      "Train Station, 7.82%\n",
      "Bus Stop, 4.73%\n",
      "Middle School, 4.52%\n",
      "Mall, 4.24%\n",
      "Platform, 3.73%\n",
      "Home (private), 2.65%\n",
      "Bus Line, 2.44%\n",
      "\n",
      "Cluster 13:\n",
      "Factory, 17.09%\n",
      "Bar, 16.34%\n",
      "TV Station, 11.42%\n",
      "Pub, 8.70%\n",
      "Park, 3.90%\n",
      "Nightclub, 2.61%\n",
      "Home (private), 2.23%\n",
      "Video Game Store, 2.16%\n",
      "Speakeasy, 2.08%\n",
      "\n",
      "Cluster 14:\n",
      "City, 26.76%\n",
      "Country, 8.79%\n",
      "Neighborhood, 7.13%\n",
      "Park, 4.99%\n",
      "Street, 3.06%\n",
      "Plaza, 2.81%\n",
      "Hotel, 2.20%\n",
      "County, 2.14%\n",
      "Residential Building (Apartment / Condo), 1.86%\n",
      "\n",
      "Cluster 15:\n",
      "Gym, 41.29%\n",
      "Coffee Shop, 5.90%\n",
      "Cultural Center, 2.86%\n",
      "Theater, 2.39%\n",
      "Café, 2.16%\n",
      "Art Museum, 1.90%\n",
      "Medical Center, 1.82%\n",
      "Multiplex, 1.82%\n",
      "Sushi Restaurant, 1.79%\n",
      "\n",
      "Cluster 16:\n",
      "Gym / Fitness Center, 7.41%\n",
      "Park, 5.28%\n",
      "Coffee Shop, 5.20%\n",
      "Mall, 4.75%\n",
      "Café, 3.22%\n",
      "Salon / Barbershop, 3.01%\n",
      "Restaurant, 3.00%\n",
      "Hospital, 2.90%\n",
      "Multiplex, 2.85%\n",
      "\n",
      "Cluster 17:\n",
      "Coworking Space, 10.34%\n",
      "Housing Development, 9.21%\n",
      "Residential Building (Apartment / Condo), 4.19%\n",
      "Building, 3.51%\n",
      "Street, 2.84%\n",
      "Police Station, 2.84%\n",
      "Gas Station / Garage, 2.75%\n",
      "Office, 2.53%\n",
      "Hotel, 2.49%\n",
      "\n",
      "Cluster 18:\n",
      "University, 26.26%\n",
      "Mall, 8.69%\n",
      "College Residence Hall, 5.96%\n",
      "Park, 5.57%\n",
      "Residential Building (Apartment / Condo), 3.69%\n",
      "Home (private), 2.97%\n",
      "Plaza, 2.79%\n",
      "Neighborhood, 2.23%\n",
      "College Academic Building, 2.17%\n",
      "\n",
      "Cluster 19:\n",
      "Office, 62.50%\n",
      "Park, 3.72%\n",
      "Mall, 3.11%\n",
      "Coffee Shop, 2.44%\n",
      "Residential Building (Apartment / Condo), 1.94%\n",
      "Train Station, 1.62%\n",
      "Multiplex, 1.59%\n",
      "Military Base, 1.38%\n",
      "Sushi Restaurant, 1.30%\n",
      "\n",
      "Cluster 20:\n",
      "Residential Building (Apartment / Condo), 35.57%\n",
      "Mall, 5.08%\n",
      "Building, 4.33%\n",
      "Park, 3.76%\n",
      "Street, 3.03%\n",
      "Train Station, 2.76%\n",
      "Neighborhood, 2.76%\n",
      "Plaza, 2.66%\n",
      "Village, 1.94%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "n_top_venues = 10\n",
    "for i, venues_dist in enumerate(topic_cats):\n",
    "    indicies = np.argsort(venues_dist)[:-n_top_venues:-1]\n",
    "    topic_venues = np.array(categoriesVocab)[indicies]\n",
    "    topic_probs = venues_dist[indicies]\n",
    "    print('Cluster {}:'.format(i+1))\n",
    "    for categoryId, topicProb in zip(topic_venues, topic_probs):\n",
    "        if topicProb>=0.01:\n",
    "            print(\"{}, {:.2f}%\").format(categoryNameFromId(categoryId),topicProb*100)\n",
    "        else:\n",
    "            break\n",
    "    print ''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 228,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4\n",
      "140\n"
     ]
    }
   ],
   "source": [
    "print len(set(multiUserCheckins[multiUserCheckins.userId == clusters[10][1]]['venueId'].tolist()).intersection(multiUserCheckins[multiUserCheckins.userId == clusters[10][2]]['venueId'].tolist()))\n",
    "print len(set(multiUserCheckins[multiUserCheckins.userId == clusters[10][1]]['venueId'].tolist()).union(multiUserCheckins[multiUserCheckins.userId == clusters[10][2]]['venueId'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015-07-10 00:56:53.189411\n",
      "2015-07-10 00:58:12.707836\n",
      "2015-07-10 00:58:14.715938\n",
      "2015-07-10 00:58:16.722812\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-bfdad1bb8258>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mwhile\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0mschedule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_pending\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 9\u001b[0;31m     \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msleep\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "import schedule\n",
    "import time\n",
    "\n",
    "schedule.clear()\n",
    "schedule.every(2).seconds.do(saveDate)\n",
    "\n",
    "while True:\n",
    "    schedule.run_pending()\n",
    "    time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def saveDate():\n",
    "    global currentDate\n",
    "    print currentDate\n",
    "    currentDate = datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "currentDate = datetime.now()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "str"
      ]
     },
     "execution_count": 181,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(allCheckins[:2][\"createdAtDate\"][0]\n",
    "     from datetime import datetime\n",
    "def extractComponents(row):\n",
    "    date = datetime.utcfromtimestamp(row.createdAt+180*60)\n",
    "    hours.append(date.hour)\n",
    "    weekdays.append(date.weekday())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code for Visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 281,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#Make lists of users check-in venues\n",
    "groupedUsersList = allCheckins.groupby('userId')\n",
    "TMUserVenuesList = []\n",
    "TMUsersCategoriesList = []\n",
    "TMUserIds = []\n",
    "\n",
    "for index, group in enumerate(groupedUsersList):\n",
    "    sortedCheckins = group[1].sort(\"createdAt\")\n",
    "    venuePairs = []\n",
    "    categoryPairs = []\n",
    "    for venue1, venue2 in pairwise(sortedCheckins['venueId'].tolist()):\n",
    "        venuePairs.append((venue1, venue2))\n",
    "        categoryPairs.append((venueCategoryFromId(venue1),venueCategoryFromId(venue2)))\n",
    "    timeDiffs = []\n",
    "    for checkin1, checkin2 in pairwise(sortedCheckins['createdAt'].tolist()):\n",
    "        timeDiffs.append((checkin2-checkin1)/60)\n",
    "    TMUserVenuesList.append(zip(venuePairs,timeDiffs))\n",
    "    TMUsersCategoriesList.append(zip(categoryPairs,timeDiffs))\n",
    "    TMUserIds.append(group[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 496,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from sys import stdout\n",
    "def progress(i, n):\n",
    "    stdout.write(\"\\r%f%%\" % (i*100/float(n)))\n",
    "    stdout.flush()\n",
    "    if i == n-1:\n",
    "        stdout.write(\"\\r100%\")\n",
    "        print(\"\\r\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 518,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getSequencesFromCheckins(checkinsDF):\n",
    "    checkinsGroupedByUser = checkinsDF.groupby('userId')\n",
    "    venuePairsGroupedByUser = []\n",
    "    categoryPairsGroupedByUser = []\n",
    "    userIds = []\n",
    "    count = len(checkinsGroupedByUser)\n",
    "\n",
    "    for index, group in enumerate(checkinsGroupedByUser):\n",
    "        progress(index, count)\n",
    "        sortedCheckins = group[1].sort(\"createdAt\")\n",
    "        \n",
    "        venuePairs = []\n",
    "        categoryPairs = []\n",
    "        \n",
    "        for venue1, venue2 in pairwise(sortedCheckins['venueId'].tolist()):\n",
    "            venuePairs.append((venue1, venue2))\n",
    "            categoryPairs.append((venueCategoryFromId(venue1), venueCategoryFromId(venue2)))\n",
    "            \n",
    "        timeDiffs = []\n",
    "        for checkin1, checkin2 in pairwise(sortedCheckins['createdAt'].tolist()):\n",
    "            timeDiffs.append((checkin2-checkin1)/60)\n",
    "            \n",
    "        venuePairsGroupedByUser.append(zip(venuePairs,timeDiffs))\n",
    "        categoryPairsGroupedByUser.append(zip(categoryPairs,timeDiffs))\n",
    "        userIds.append(group[0])\n",
    "    return (venuePairsGroupedByUser, categoryPairsGroupedByUser, userIds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "(venuePairsGroupedByUser5, categoryPairsGroupedByUser5, userIds5) = getSequencesFromCheckins(LDAclusteredUsersDF[LDAclusteredUsersDF.clusterId == 5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "averageTimes = [np.mean([transitionTuple[1] for transitionTuple in userTransitions])/60 for userTransitions in TMUserVenuesList]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 533,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import collections\n",
    "def getCategoryPredcessorsCounter(categoryName, categoryPairs):\n",
    "    predcessorsByUser = [[transitionTuple[0][0] for transitionTuple in userTransitions if transitionTuple[0][1] == categoryName and transitionTuple[1] < 60*6] for userTransitions in categoryPairs]\n",
    "    predcessorsList = [item for sublist in predcessorsByUser for item in sublist]\n",
    "    return collections.Counter(predcessorsList)\n",
    "\n",
    "def getCategorySuccessorsCounter(categoryName, categoryPairs):\n",
    "    successorsByUser = [[transitionTuple[0][1] for transitionTuple in userTransitions if transitionTuple[0][0] == categoryName and transitionTuple[1] < 60*6] for userTransitions in categoryPairs]\n",
    "    successorsList =  [item for sublist in successorsByUser for item in sublist]\n",
    "    return collections.Counter(successorsList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "categoryCounter5 = getCategoryPredcessorsCounter('Coffee Shop', categoryPairsGroupedByUser5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 565,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "saveTopSuccessorsColorsJSON('Coffee Shop', categoryPairsGroupedByUser5, 'Production/vis/colors5.json', 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 564,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getTopSuccessorsNames(categoryName, categoryPairs, nTop = 10):\n",
    "    topNames = [commonTuple[0] for commonTuple in  getCategoryPredcessorsCounter(categoryName, categoryPairs).most_common(nTop)]\n",
    "    return topNames\n",
    "def saveTopSuccessorsColorsCSV(categoryName, categoryPairs, filename, nTop = 10):\n",
    "    topNames = getTopSuccessorsNames(categoryName, categoryPairs, nTop)\n",
    "    nameColorList = pd.DataFrame({'name':topNames,\n",
    "    'color':[\n",
    "    '#3182bd',\n",
    "    '#e6550d',\n",
    "    '#fdae6b',\n",
    "    '#31a354',\n",
    "    '#756bb1',\n",
    "    '#636363',\n",
    "    '#d6616b',\n",
    "    '#7b4173',\n",
    "    '#bd9e39',\n",
    "    #'#b5cf6b',\n",
    "    '#a1d99b']})\n",
    "    nameColorList.to_csv(filename, encoding=\"utf-8\", index=None)\n",
    "    \n",
    "def saveTopSuccessorsColorsJSON(categoryName, categoryPairs, filename, nTop = 10):\n",
    "    topNames = getTopSuccessorsNames(categoryName, categoryPairs, nTop)\n",
    "    jsonOfColors = [dict(zip(['name', 'color'],row)) for row in izip(topNames,[\n",
    "    '#3182bd',\n",
    "    '#e6550d',\n",
    "    '#fdae6b',\n",
    "    '#31a354',\n",
    "    '#756bb1',\n",
    "    '#636363',\n",
    "    '#d6616b',\n",
    "    '#7b4173',\n",
    "    '#bd9e39',\n",
    "    #'#b5cf6b',\n",
    "    '#a1d99b'])]\n",
    "    with open(filename, 'w') as outfile:\n",
    "        json.dump(jsonOfColors, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def getMatrixForTopSuccessors(categoryName, categoryPairs, nTop = 10, withOthers = 1):\n",
    "    #Get the counter and list of successors\n",
    "    categoryCounter = getCategorySuccessorsCounter(categoryName, categoryPairs)\n",
    "    categoryTopSuccessors = categoryCounter.most_common(nTop)\n",
    "    topSuccessorsNames = [commonTuple[0] for commonTuple in categoryTopSuccessors]\n",
    "    \n",
    "    #Calculate total sums of inputs/outputs\n",
    "    totalOut = sum(categoryCounter.values())\n",
    "    totalTopCatsInterOut = 0\n",
    "    totalIn = sum(getCategoryPredcessorsCounter(categoryName, categoryPairs).values())\n",
    "    \n",
    "    #+1 for Others section\n",
    "    adjMatrix = [[] for i in range(nTop + withOthers)]\n",
    "    \n",
    "    #Fill the first element with the self value\n",
    "    adjMatrix[0] = [0 for i in range(nTop + withOthers)]\n",
    "    adjMatrix[0][0] = categoryCounter[categoryName]\n",
    "    \n",
    "    index = 1\n",
    "    for commonTuple in  categoryTopSuccessors:\n",
    "        successorCategory = commonTuple[0]\n",
    "        if successorCategory != categoryName:\n",
    "            #Count of transitions from source category\n",
    "            adjMatrix[0][index] = commonTuple[1]\n",
    "            \n",
    "            successorCategoryCounter = getCategorySuccessorsCounter(successorCategory, categoryPairs)\n",
    "            #Counts for all the important categories\n",
    "            adjRow = [0 for i in range(nTop)]\n",
    "            adjRow[0] = successorCategoryCounter[categoryName]\n",
    "            adjRow[1:] = [successorCategoryCounter[catName] for catName in topSuccessorsNames if catName!=categoryName]\n",
    "            #Others set to 0\n",
    "            if withOthers:\n",
    "                adjRow.append(0)\n",
    "            adjMatrix[index] = adjRow\n",
    "            totalTopCatsInterOut  = totalTopCatsInterOut + sum(adjRow)\n",
    "            index = index + 1\n",
    "    if withOthers:\n",
    "        adjMatrix[0][nTop] = totalOut - sum(adjMatrix[0])\n",
    "        adjMatrix[nTop] = [0 for i in range(nTop + withOthers)]\n",
    "        adjMatrix[nTop][0] = totalIn - sum(row[0] for row in adjMatrix)\n",
    "    \n",
    "    #Normalize\n",
    "    total = sum([sum(sublist) for sublist in adjMatrix])\n",
    "    normMatrix = [[float(count)/total for count in adjRow] for adjRow in adjMatrix]\n",
    "    return normMatrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('Vis/coffeeMatrix5.json', 'w') as outfile:\n",
    "    json.dump(getMatrixForTopSuccessors('Coffee Shop', categoryPairsGroupedByUser5, 10, 0), outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "averageTimesFiltered = [averageTime for averageTime in averageTimes if averageTime < 24*7]\n",
    "print int(float(len(averageTimesFiltered))/len(averageTimes)*100),\"%\"\n",
    "plt.hist(averageTimesFiltered, bins = 48-0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "averageTimesFiltered = [averageTime for averageTime in averageTimes if averageTime >= 24*7 and averageTime <= 24*30]\n",
    "print int(float(len(averageTimesFiltered))/len(averageTimes)*100),\"%\"\n",
    "plt.hist(averageTimesFiltered, bins = 30-0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 283,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('data/categoryPairs.json', 'w') as outfile:\n",
    "    json.dump(TMUsersCategoriesList, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from itertools import tee, izip\n",
    "def pairwise(iterable):\n",
    "    a, b = tee(iterable)\n",
    "    next(b, None)\n",
    "    return izip(a, b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(allCheckins[['venueId', 'categoryId']].set_index('venueId').to_dict().keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
       "       18, 19, 20])"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(LDAclusteredUsersDF['clusterId'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "LNG = allCheckins[['venueId', 'lat','lng']].set_index('venueId').to_dict()['lng']\n",
    "LAT = allCheckins[['venueId', 'lat','lng']].set_index('venueId').to_dict()['lat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from itertools import izip\n",
    "for (i, j), (k, v) in izip(LNG.iteritems(), LAT.iteritems()):\n",
    "    print i, j, k, v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "allCheckins[['venueId', 'categoryId']].set_index('venueId').to_dict()['categoryId']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([u'categoryId', u'categoryName', u'checkinId', u'checkinsCount',\n",
       "       u'createdAt', u'createdAtDate', u'lat', u'lng', u'name', u'tipCount',\n",
       "       u'userId', u'usersCount', u'venueId'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "allCheckins.keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "checkinsCountGrouped = []\n",
    "for group in LDAclusteredUsersDF[LDAclusteredUsersDF.clusterId == 1].groupby(['hour','weekday']):\n",
    "    checkinsCountGrouped.append({'key':group[0],\n",
    "                               'value':len(group[1])})\n",
    "checkinsCountGrouped"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('testTime.json', 'w') as outfile:\n",
    "        json.dump(checkinsCountGrouped, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "topCheckins = pd.read_csv('data/July/LDAClusteredUsersMoreThan10.tsv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "56477"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(topCheckins['name'].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set(topCheckins['clusterId'].tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'4b08d1d7f964a520a11223e3': 13,\n",
       " '4b4d5b48f964a52055d126e3': 5,\n",
       " '4b4f33fff964a52024fd26e3': 5,\n",
       " '4b5086ccf964a520212627e3': 18,\n",
       " '4b57605cf964a520873428e3': 11,\n",
       " '4b62c18df964a52069512ae3': 18,\n",
       " '4b63406af964a520a36d2ae3': 20,\n",
       " '4b645287f964a52095aa2ae3': 9,\n",
       " '4b71a2a7f964a52095522de3': 14,\n",
       " '4b7563d3f964a520d8092ee3': 13,\n",
       " '4b76c7bdf964a520a85e2ee3': 8,\n",
       " '4b76e9d3f964a520e7692ee3': 20,\n",
       " '4b827a36f964a52098d430e3': 18,\n",
       " '4b9379e6f964a520364534e3': 20,\n",
       " '4b937e60f964a520e34534e3': 6,\n",
       " '4b9bb751f964a520361d36e3': 11,\n",
       " '4b9cb499f964a520617736e3': 13,\n",
       " '4b9ce827f964a520b48136e3': 7,\n",
       " '4b9f2998f964a520161637e3': 13,\n",
       " '4ba1ba30f964a520cac937e3': 9,\n",
       " '4ba3a1c7f964a520064e38e3': 12,\n",
       " '4ba60394f964a520582e39e3': 9,\n",
       " '4ba7142cf964a520aa7f39e3': 20,\n",
       " '4ba754cdf964a520268c39e3': 20,\n",
       " '4ba765e4f964a520b09039e3': 9,\n",
       " '4ba8486cf964a520d7d439e3': 9,\n",
       " '4ba9c796f964a520c5373ae3': 20,\n",
       " '4bab6062f964a520a3a43ae3': 12,\n",
       " '4bab69a0f964a5206aa73ae3': 9,\n",
       " '4bab748cf964a520aeaa3ae3': 12,\n",
       " '4bade2cbf964a52006703be3': 6,\n",
       " '4bade94ef964a520f9713be3': 20,\n",
       " '4bb1f5c0f964a5209aad3ce3': 11,\n",
       " '4bb3345942959c74d79d212c': 19,\n",
       " '4bb41c28f187a593f1e913f8': 11,\n",
       " '4bb73b3b1344b71317789e04': 8,\n",
       " '4bb75b522ea1952199cdac2f': 13,\n",
       " '4bb8469a1261d13a5586e898': 10,\n",
       " '4bbb7289e6e7b7139a10f879': 19,\n",
       " '4bbdf078078095211927db91': 14,\n",
       " '4bbea7ae006dc9b6d096fb3f': 20,\n",
       " '4bbf37d430c99c74bfc75411': 12,\n",
       " '4bbf8718b492d13a0acea260': 20,\n",
       " '4bc005a02a89ef3b8113f088': 12,\n",
       " '4bc04797920eb7130bc0182c': 20,\n",
       " '4bc06ccfabf495219932bf93': 7,\n",
       " '4bc090ca920eb7132e14192c': 18,\n",
       " '4bc09d91461576b055617a32': 18,\n",
       " '4bc15f44b492d13ab312a660': 1,\n",
       " '4bc1a90e4cdfc9b6e2ef9421': 9,\n",
       " '4bc44534920eb7132c7d1e2c': 2,\n",
       " '4bc4f2f80191c9b6c7e0eab1': 17,\n",
       " '4bc5594d6a3e9c744bd3f548': 15,\n",
       " '4bc6e22e0050b7138eb3b73b': 11,\n",
       " '4bc6f0fdaf07a593563f7e2d': 20,\n",
       " '4bc8760792b376b03767513a': 14,\n",
       " '4bc9559768f976b028465d83': 1,\n",
       " '4bc9bc9bb6c49c74bdd88e91': 20,\n",
       " '4bc9c2420687ef3b3b64dacc': 13,\n",
       " '4bcaaf463740b713aa586165': 13,\n",
       " '4bcc74e868f976b0b5996283': 7,\n",
       " '4bcd4510fb84c9b65ee0213e': 9,\n",
       " '4bd1cb2c41b9ef3b6f94fce5': 20,\n",
       " '4bd2739da8b3a5935010685f': 7,\n",
       " '4bd32c46b221c9b6c107d9d0': 12,\n",
       " '4bd335bc41b9ef3bbe8dffe5': 14,\n",
       " '4bd704c729eb9c74edfb95e1': 9,\n",
       " '4bd714b05631c9b641e6a630': 13,\n",
       " '4bd83bdadc4b9521169e7888': 5,\n",
       " '4bd8749835aad13a99eb90f3': 13,\n",
       " '4bd996992e6f0f47a7170b08': 8,\n",
       " '4bd9ae62905bef3bc8376e93': 19,\n",
       " '4bdc8a553904a59337a84e9e': 13,\n",
       " '4bdc9716afe8c9b689235085': 18,\n",
       " '4bdcdd1eafe8c9b6e4605185': 17,\n",
       " '4bdd5f27f219c9b6bb751210': 19,\n",
       " '4be17d7bc1732d7f65245c9a': 15,\n",
       " '4be198cb1a44a593d71b9287': 11,\n",
       " '4be3777821d5a593b15c1911': 6,\n",
       " '4be425c9bcef2d7f471e02e5': 10,\n",
       " '4be6909fd4f7c9b6b2e02620': 19,\n",
       " '4be6ce08910020a1bb9fd414': 15,\n",
       " '4be6d5f6d4f7c9b6af172720': 17,\n",
       " '4be7f29dc5220f47fc16a9ca': 6,\n",
       " '4be82d4488ed2d7fde85cb1d': 14,\n",
       " '4bf44f352d629521081b6058': 9,\n",
       " '4bf79c2e4a67c928f1f223cf': 7,\n",
       " '4bf7d2d55efe2d7fcf8d6934': 17,\n",
       " '4bf7fa734a67c928e27624cf': 8,\n",
       " '4bf8697492d19521313a5a1f': 13,\n",
       " '4bfb86c2f214a5930e4eaf0e': 6,\n",
       " '4bfc9df283bbd13aa4d462c5': 18,\n",
       " '4c00f61b9a950f470e5808c6': 17,\n",
       " '4c0120da360a9c74a716d9a0': 11,\n",
       " '4c0124bb19d8c92811528829': 11,\n",
       " '4c063c08761ac9b675972074': 13,\n",
       " '4c0772808a81c9b687392790': 14,\n",
       " '4c087466009a0f47473ee6bf': 14,\n",
       " '4c088729009a0f478a58e6bf': 8,\n",
       " '4c08dbfca1b32d7fe59d96f0': 7,\n",
       " '4c0a51027e3fc9289c55f382': 5,\n",
       " '4c0a5a89bbc676b0c99f49d5': 20,\n",
       " '4c0b65a2bbc676b003864bd5': 2,\n",
       " '4c0bbdb76071a593e5a2e132': 3,\n",
       " '4c0c8bacc700c9b6b81ea1dd': 15,\n",
       " '4c11ccd2416620a1cfced9e4': 20,\n",
       " '4c11f318d5599521185818b9': 18,\n",
       " '4c125c95b2510f47fb78c498': 9,\n",
       " '4c14a2547f7f2d7fe036e168': 6,\n",
       " '4c1769a8216fc9b6a8558f96': 13,\n",
       " '4c17a3fe30d30f4751ce36a9': 20,\n",
       " '4c1a49ddc09ed13a4b47828e': 8,\n",
       " '4c1ce5e6fcf8c9b673c0aa0b': 15,\n",
       " '4c1dde9ab306c928913f66b7': 14,\n",
       " '4c247fecf1272d7f5e1983c5': 17,\n",
       " '4c260a345c5ca593be5745fe': 17,\n",
       " '4c2c408c8abca5930e7c0120': 13,\n",
       " '4c2c7cc4d1a10f473d7bf964': 20,\n",
       " '4c2e329c1683b713218c26ec': 20,\n",
       " '4c34ecc53ffc9521e60791f5': 8,\n",
       " '4c35c57616adc9284c3bc79c': 19,\n",
       " '4c3f4508e26920a1aa2f5be7': 11,\n",
       " '4c49ca906f1420a125001a56': 20,\n",
       " '4c55a14772cf0f47a72493d6': 15,\n",
       " '4c5d5c9e2815c9289683b467': 14,\n",
       " '4c6a53d5897b1b8d947ab117': 8,\n",
       " '4c6cd8d7fe77bfb7ff74a023': 9,\n",
       " '4c6d4302f2846dcba80a65e9': 17,\n",
       " '4c809bfa2f1c236ac85a2843': 11,\n",
       " '4c8ba2663dc2a1cd81eeb732': 9,\n",
       " '4c921f6c57e5b60cc629661c': 17,\n",
       " '4c98db097b9d37047ab6e185': 9,\n",
       " '4c9ca95c9975a143b40334cf': 8,\n",
       " '4cb17cc975ebb60ca2aacdad': 7,\n",
       " '4cb47dcb75ebb60c13e7e2ad': 19,\n",
       " '4cc1194caa5776b0a1bc7544': 1,\n",
       " '4ccf90921ac7a1cd9e7d1492': 19,\n",
       " '4cd59a2e886cb60cff068589': 13,\n",
       " '4cecd7837b943704a3302353': 9,\n",
       " '4cee97333b03f04da5ba3adc': 2,\n",
       " '4cf0cbbd1c158cfa209accb5': 17,\n",
       " '4cf235528333224bb0150f8e': 8,\n",
       " '4cf622e2e308236ab65ee4ac': 13,\n",
       " '4d00cfa17c563704f055b2f0': 5,\n",
       " '4d01c5cd68e38eec0fb2dec4': 1,\n",
       " '4d048e4926adb1f70367d370': 11,\n",
       " '4d2afbf0ebacb1f7c4fa0750': 17,\n",
       " '4d3340506c7c721e00a5b756': 7,\n",
       " '4d468407bf61a1cd34441dac': 6,\n",
       " '4d600671b19fa143a476efc8': 14,\n",
       " '4d6a4059de28224b34854dbe': 12,\n",
       " '4d7630626c22b60c4cade4d6': 6,\n",
       " '4d7b42a3f260a093113124ba': 20,\n",
       " '4d937470922c6ea8106e6679': 19,\n",
       " '4d95725397d06ea8d57eeb0a': 8,\n",
       " '4d99485ae07ea35d201b0603': 6,\n",
       " '4d9abff1422ea1cd52d9ec4c': 17,\n",
       " '4da15927bf22a143c42c9af8': 12,\n",
       " '4da98cf46e81162ae7bf3556': 10,\n",
       " '4dbad93743a1d8504b5f19c7': 15,\n",
       " '4dbec57a0cb691071cd71859': 6,\n",
       " '4dc6a261b0fb5556cd0dec37': 7,\n",
       " '4dda4e49d4c05d50972cfa13': 5,\n",
       " '4de115cc22713271e2c3bdfd': 6,\n",
       " '4df48a02483bdb5e1378145c': 17,\n",
       " '4e03363645ddd36826987d78': 14,\n",
       " '4e0776da1f6e2110338d2afc': 10,\n",
       " '4e290e1c149554c77440442a': 15,\n",
       " '4e30644188778782371f9bf2': 20,\n",
       " '4e36f254d22d86185a7b93da': 15,\n",
       " '4e3bfe241838cb1b82db6787': 12,\n",
       " '4e4aab0f62e13240b700e034': 20,\n",
       " '4e5b2a2022718867150a1295': 3,\n",
       " '4e5b506081dcc95bf501eb47': 14,\n",
       " '4e6dba02b61cf2b03c0b679a': 11,\n",
       " '4e735693a8095a7c423cecc4': 7,\n",
       " '4e933e2355037de99da8c992': 14,\n",
       " '4eb032e74690488dac054570': 8,\n",
       " '4ee476d1be7b4845a4510ca8': 6,\n",
       " '4f01c7048231bcb70598c35d': 14,\n",
       " '4f16ed94e4b0044a28ac6094': 10,\n",
       " '4f254b58e4b067a327a9f316': 7,\n",
       " '4f366bc5e4b0571dc65225f8': 9,\n",
       " '4f38e7dde4b0edf91e749221': 17,\n",
       " '4f3b4fd9e4b0cdda8406cf52': 8,\n",
       " '4f3b52bfe4b0ce0258fcafd4': 12,\n",
       " '4f3fbb40e4b0f13aa647ef54': 7,\n",
       " '4f412b63e4b020398d5b9282': 8,\n",
       " '4f5c7970e4b0c5418ba80278': 16,\n",
       " '4f5d9f87e4b070e048e580e8': 14,\n",
       " '4f9328fee4b0b2dfd05c8fe5': 19,\n",
       " '4f9416a6e4b0c98ce115c907': 15,\n",
       " '4f9abd03e4b0de6266a6bc33': 15,\n",
       " '4f9ff91de4b05d0fc0fab715': 15,\n",
       " '4fa408ade4b039aeea528316': 9,\n",
       " '4fc76ffee4b0c5a82f3dd0e4': 10,\n",
       " '503206e5e4b0f8a001fc1b78': 20,\n",
       " '50371cace4b0fc35e33045ae': 5,\n",
       " '505342e5e4b09a662c8bb026': 15,\n",
       " '5082ba6de4b013424915883f': 15,\n",
       " '50d02ad2e4b0a13d85ae5a03': 12,\n",
       " '511b936be4b0acbf01fefe86': 6,\n",
       " '519de322498e0634f8ebc2f5': 7,\n",
       " '51d571de498e56324e7df2c5': 13,\n",
       " '520ca7b311d2feb68b513bd1': 17,\n",
       " '520f117111d2a782f1211c3b': 19,\n",
       " '522482e811d2c23bb3190e1f': 12,\n",
       " '527fd41311d2ac6c3122415d': 7,\n",
       " '53110e9be4b03443dd848438': 16,\n",
       " '531f3474498ec1d12e0aaa10': 9,\n",
       " '5348ef89498ed70ddb33165c': 19,\n",
       " '538c981b498e254777d000e8': 7,\n",
       " '53d57fc1498e5bc57a00d845': 16,\n",
       " '5419b1d0498edb50e2690c1e': 6,\n",
       " '54257ee8498e1c43a2843359': 7,\n",
       " '543d457b498e78381ac8bd67': 20,\n",
       " '543e9640498eef542d4c18b1': 10,\n",
       " '54808b0a498e531aaca28796': 12,\n",
       " '5486d678498e4eca5a4fea8b': 5,\n",
       " '5489b095498e001dde26a4ae': 12,\n",
       " '55253d8a498e00251f47f6ef': 12}"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "topCheckins[['venueId', 'clusterId']].set_index('venueId').to_dict()['clusterId']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "venueByCluster = [0 for i in range(20)]\n",
    "for clusterCheckins in topCheckins.groupby('clusterId'):\n",
    "    names =  list(np.unique(clusterCheckins[1]['name'].tolist()))\n",
    "    ids = list(np.unique(clusterCheckins[1]['venueId'].tolist()))\n",
    "    venueByCluster[clusterCheckins[0] - 1] = {\"names\": names,\n",
    "                                         \"ids\":ids}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('data/clustersTopVenues.json', 'w') as outfile:\n",
    "        json.dump(venueByCluster, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n",
      "20\n"
     ]
    }
   ],
   "source": [
    "categoryByCluster = [[] for i in range(20)]\n",
    "for clusterCheckins in topCheckins.groupby('clusterId'):\n",
    "    print clusterCheckins[0]\n",
    "    for categoryId, group in clusterCheckins[1].groupby('categoryName').agg('size').order(ascending = False).iteritems():\n",
    "        categoryByCluster[clusterCheckins[0]-1].append(categoryId)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for index, category in enumerate(categoryByCluster):\n",
    "    categoryByCluster[index] = categoryByCluster[index][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "with open('data/clustersTopCategories.json', 'w') as outfile:\n",
    "        json.dump(categoryByCluster, outfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}