jupyter_notebooks/I_probability_hw_2_naive_bayes.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Реализация наивного Байесовского классификатора"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>sms_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  class                                           sms_text\n",
       "0  spam  Free entry in 2 a wkly comp to win FA Cup fina...\n",
       "1   ham  U dun say so early hor... U c already then say...\n",
       "2   ham  Nah I don't think he goes to usf, he lives aro..."
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "filename = 'data/sms_spam_collection.tar.gz'\n",
    "\n",
    "df = pd.read_csv(\n",
    "    filename,\n",
    "    compression='gzip',\n",
    "    header=1,\n",
    "    sep='\\t',\n",
    "    encoding='utf8',\n",
    "    names=['class', 'sms_text'],\n",
    "    error_bad_lines=False\n",
    ")\n",
    "\n",
    "\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Проверяем, сколько у нас всего объектов в датасете"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5571 2\n"
     ]
    }
   ],
   "source": [
    "num_objects, num_features = df.shape\n",
    "print(num_objects, num_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Целевая переменная (target) в столбце `class`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    spam\n",
       "1     ham\n",
       "2     ham\n",
       "3    spam\n",
       "4     ham\n",
       "Name: class, dtype: object"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['class'].head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Демонстрация того, как получить булеву маску для датафрейма"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        True\n",
       "1       False\n",
       "2       False\n",
       "3        True\n",
       "4       False\n",
       "        ...  \n",
       "5566    False\n",
       "5567    False\n",
       "5568    False\n",
       "5569    False\n",
       "5570    False\n",
       "Name: class, Length: 5571, dtype: bool"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "SPAM_CLASS = 'spam'\n",
    "NOT_SPAM_CLASS = 'ham'\n",
    "\n",
    "df['class'] == SPAM_CLASS"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Использование булевой маски для фильтрации датафрейма "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spam sms: 747, not spam sms 4823\n"
     ]
    }
   ],
   "source": [
    "spam_sms_num = (df['class'] == SPAM_CLASS).sum()\n",
    "notspam_sms_num = (df['class'] == NOT_SPAM_CLASS).sum()\n",
    "\n",
    "print(f'spam sms: {spam_sms_num}, not spam sms {notspam_sms_num}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Пример - считаем вероятности классов"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1341, 0.8657\n"
     ]
    }
   ],
   "source": [
    "# априорная вероятность класса спам\n",
    "p_spam = spam_sms_num / num_objects\n",
    "\n",
    "# априорная вероятность класса не спам\n",
    "p_notspam = notspam_sms_num / num_objects\n",
    "\n",
    "print(f'{p_spam:.4f}, {p_notspam:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Пример обработки текстовой информации - приводим к нижнему регистру"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'free'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_word = 'Free'.lower()\n",
    "\n",
    "test_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Go until jurong point, crazy.. Available only in bugis n great world la e buffet... Cine there got amore wat...'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sms_example = df['sms_text'].values[0]\n",
    "\n",
    "sms_example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.1341, 0.8659\n"
     ]
    }
   ],
   "source": [
    "# априорная вероятность класса спам\n",
    "p_spam = spam_sms_num / num_objects\n",
    "\n",
    "# априорная вероятность класса не спам\n",
    "p_notspam = notspam_sms_num / num_objects\n",
    "\n",
    "print(f'{p_spam:.4f}, {p_notspam:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Пишем полезные сниппеты для трансформации текста"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'Go until jurong point crazy Available only in bugis n great world la e buffet Cine there got amore wat'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# удаляем знаки препинания\n",
    "import string\n",
    "\n",
    "print(string.punctuation)\n",
    "\n",
    "sms_example = ''.join([\n",
    "    char\n",
    "    for char in sms_example\n",
    "    if char not in string.punctuation\n",
    "])\n",
    "\n",
    "sms_example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'go until jurong point crazy available only in bugis n great world la e buffet cine there got amore wat'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# приводим слова к нижнему регистру\n",
    "\n",
    "#.lower()\n",
    "sms_example = ' '.join([\n",
    "    word.lower()\n",
    "    for word in sms_example.split(sep=' ')\n",
    "])\n",
    "\n",
    "sms_example"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Объединяем сниппеты в функцию"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "free entry in 2 a wkly comp to win fa cup final tkts 21st may 2005 text fa to 87121 to receive entry questionstd txt ratetcs apply 08452810075over18s\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "def text_preprocess(sms_text: str) -> str:\n",
    "    \"\"\"Преобразование текста для анализа\"\"\"\n",
    "    text_no_punctuation = ''.join([\n",
    "        char\n",
    "        for char in sms_text\n",
    "        if char not in string.punctuation\n",
    "    ])\n",
    "    text_lowercase = ' '.join([\n",
    "        word.lower()\n",
    "        for word in text_no_punctuation.split(sep=' ')\n",
    "    ])\n",
    "    \n",
    "    return text_lowercase\n",
    "\n",
    "\n",
    "sms_example = df['sms_text'].values[0]\n",
    "\n",
    "print(text_preprocess(sms_example))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Трансформируем каждую строчку датафрейма"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>class</th>\n",
       "      <th>sms_text</th>\n",
       "      <th>processed_text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ham</td>\n",
       "      <td>Go until jurong point, crazy.. Available only ...</td>\n",
       "      <td>go until jurong point crazy available only in ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ham</td>\n",
       "      <td>Ok lar... Joking wif u oni...</td>\n",
       "      <td>ok lar joking wif u oni</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>spam</td>\n",
       "      <td>Free entry in 2 a wkly comp to win FA Cup fina...</td>\n",
       "      <td>free entry in 2 a wkly comp to win fa cup fina...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ham</td>\n",
       "      <td>U dun say so early hor... U c already then say...</td>\n",
       "      <td>u dun say so early hor u c already then say</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ham</td>\n",
       "      <td>Nah I don't think he goes to usf, he lives aro...</td>\n",
       "      <td>nah i dont think he goes to usf he lives aroun...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  class                                           sms_text  \\\n",
       "0   ham  Go until jurong point, crazy.. Available only ...   \n",
       "1   ham                      Ok lar... Joking wif u oni...   \n",
       "2  spam  Free entry in 2 a wkly comp to win FA Cup fina...   \n",
       "3   ham  U dun say so early hor... U c already then say...   \n",
       "4   ham  Nah I don't think he goes to usf, he lives aro...   \n",
       "\n",
       "                                      processed_text  \n",
       "0  go until jurong point crazy available only in ...  \n",
       "1                            ok lar joking wif u oni  \n",
       "2  free entry in 2 a wkly comp to win fa cup fina...  \n",
       "3        u dun say so early hor u c already then say  \n",
       "4  nah i dont think he goes to usf he lives aroun...  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.assign(\n",
    "    processed_text=df['sms_text'].apply(text_preprocess)\n",
    ")\n",
    "\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Находим вероятность встретить слово в каждом из классов - это наша основная \"фича\" в наивном байесовском классификаторе"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "P(word=\"free\"|class=spam)=0.2664\n",
      "P(word=\"free\"|class=not_spam)=0.0137\n"
     ]
    }
   ],
   "source": [
    "# вероятность встретить слово в спам смс\n",
    "\n",
    "spam_test_word_entries = df[\n",
    "    df['class'] == SPAM_CLASS\n",
    "]['processed_text'].apply(\n",
    "    lambda row: test_word in row\n",
    ").sum()\n",
    "\n",
    "# вероятность встретить слово в не-спам смс\n",
    "notspam_test_word_entries = df[\n",
    "    df['class'] == NOT_SPAM_CLASS\n",
    "]['processed_text'].apply(\n",
    "    lambda row: test_word in row\n",
    ").sum()\n",
    "\n",
    "print(f'P(word=\"{test_word}\"|class=spam)={spam_test_word_entries/spam_sms_num:.4f}')\n",
    "print(f'P(word=\"{test_word}\"|class=not_spam)={notspam_test_word_entries/notspam_sms_num:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Вывод\n",
    "\n",
    "слово \"free\" встречается в спам смс с вероятностью $26.6\\%$, а в не-спаме с вероятностью $1.3\\%$ - т.е. это слово является хорошим \"маркером\" спама"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Реализовать классификатор\n",
    "\n",
    "Аналогично тому, как посчитали вероятности встретить слово `free` в каждом классе (спам/не спам) \n",
    "* в функции `fit()` подсчитать такие вероятности для каждого слова\n",
    "* в функции `predict()` по формуле байеса (см. лекцию) вычислять вероятность принадлежности входного текста к каждому из классов\n",
    "\n",
    "Результат предсказания - класс, вероятность принадлежности к которому больше"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "from typing import List\n",
    "from copy import deepcopy\n",
    "\n",
    "import numpy as np\n",
    "\n",
    "\"\"\"имплементация наивного байесовского классификатора\"\"\"\n",
    "class NaiveBayes:\n",
    "    def __init__(self):\n",
    "        \n",
    "        self.labels = [NOT_SPAM_CLASS, SPAM_CLASS]\n",
    "        self.class_labels_proba = None  # априорная вероятность класса\n",
    "        self.prior_word_proba = None  # частоты фичей (токенов)\n",
    "    \n",
    "    def _set_labels_prior_proba(self, data: list, target: list):\n",
    "        \"\"\"Вычисление априорной вероятности классов\"\"\"\n",
    "        class_labels_count = dict.fromkeys(self.labels, 0)\n",
    "\n",
    "        num_objects = len(data)\n",
    "        for label in target:\n",
    "            try:\n",
    "                class_labels_count[label] += 1\n",
    "            except KeyError:\n",
    "                print(f'Некорректное значение метки: {label}')\n",
    "\n",
    "        class_labels_proba = {\n",
    "            i: j / num_objects for i, j in class_labels_count.items()\n",
    "        }\n",
    "        self.class_labels_proba = class_labels_proba\n",
    "    \n",
    "    def _tokenize_text(self, text):\n",
    "        tokens = []\n",
    "        try:\n",
    "            processed_text = text_preprocess(text)\n",
    "            tokens = processed_text.split(' ')\n",
    "        except TypeError:\n",
    "            print(f'Ошибка при обработке текста sms: {text}')\n",
    "        return tokens\n",
    "    \n",
    "    def _set_word_count(self, data, target):\n",
    "        def_dict_obj = defaultdict(int)\n",
    "        word_count_dict_by_class = dict.fromkeys(self.labels)\n",
    "        for label in word_count_dict_by_class:\n",
    "            word_count_dict_by_class[label] = deepcopy(def_dict_obj)\n",
    "        word_count_dict_total = deepcopy(def_dict_obj)\n",
    "        for label, text in zip(target, data):\n",
    "                for token in self._tokenize_text(text):\n",
    "                    word_count_dict_by_class[label][token] += 1\n",
    "                    word_count_dict_total[token] += 1\n",
    "        # переходим от счётчиков к частотам\n",
    "        total_token_count = sum((count for token, count in word_count_dict_total.items()))\n",
    "        print(f'total_token_count={total_token_count}')\n",
    "        for label in word_count_dict_by_class:\n",
    "            for token in word_count_dict_by_class[label]:\n",
    "                word_count_dict_by_class[label][token] = word_count_dict_by_class[label][token] / total_token_count\n",
    "        self.prior_word_proba = word_count_dict_by_class\n",
    "\n",
    "    def fit(self, data: list, target: list):\n",
    "        \"\"\"Обучение статистик по датасету\n",
    "\n",
    "        :param data: массив документов, каждый документ - объект типа str\n",
    "        :param target: массив меток объектов\n",
    "        :return:\n",
    "        \"\"\"\n",
    "        if not isinstance(data, list):\n",
    "            raise ValueError('Аргумент data должен иметь тип list')\n",
    "        if not isinstance(target, list):\n",
    "            raise ValueError('Аргумент target должен иметь тип list')\n",
    "        print('Данные инициализированы!')\n",
    "        self._set_labels_prior_proba(data, target)\n",
    "        print(f'Априорные вероятности классов {self.class_labels_proba}')\n",
    "        self._set_word_count(data, target)\n",
    "        print('Обучили априорные вероятности слов')\n",
    "        \n",
    "\n",
    "    def _predict_proba(self, data: list) -> List[tuple]:\n",
    "        \"\"\"Предсказываем класс для текстовой смс\n",
    "\n",
    "        :param data: массив документов, для каждого из которых нужно предсказать метку\n",
    "        :return: вероятности для каждого из классов\n",
    "        \"\"\"\n",
    "        prediction = []\n",
    "        for obj in data:\n",
    "            posterior_class_proba = defaultdict(lambda: 1)\n",
    "            for token in self._tokenize_text(obj):\n",
    "                for label in self.labels:\n",
    "                    posterior_class_proba[label] *= self.prior_word_proba[label][token]\n",
    "            # сохраняем для каждой метки класса - сколько меток, таков и размер uple\n",
    "            prediction.append(\n",
    "                tuple(\n",
    "                    posterior_class_proba[label] for label in self.labels\n",
    "                )\n",
    "            )\n",
    "        print(f'proba: {prediction}')\n",
    "        return prediction\n",
    "    \n",
    "    def predict(self, data) -> List[str]:\n",
    "        predict_labels = []\n",
    "        for proba in self._predict_proba(data):\n",
    "            predict_labels.append(self.labels[np.argmax(proba)])\n",
    "        return predict_labels\n",
    "\n",
    "naive_bayes = NaiveBayes()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Данные инициализированы!\n",
      "Некорректное значение метки: nan\n",
      "Априорные вероятности классов {'ham': 0.8657332615329384, 'spam': 0.13408723747980614}\n",
      "Ошибка при обработке текста sms: nan\n",
      "total_token_count=87507\n",
      "Обучили априорные вероятности слов\n"
     ]
    }
   ],
   "source": [
    "naive_bayes.fit(\n",
    "    data=df['sms_text'].values.tolist(),\n",
    "    target=df['class'].tolist()\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.000308546744831842, 1.1427657215994148e-05)"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "naive_bayes.prior_word_proba['ham']['thank'], naive_bayes.prior_word_proba['spam']['thank']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Предсказание метки класса"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Can meh? Thgt some will clash... Really ah, i dun mind... I dun seen to have lost any weight... Gee...', 'Actually fuck that, just do whatever, do find an excuse to be in tampa at some point before january though', 'Camera - You are awarded a SiPix Digital Camera! call 09061221066 fromm landline. Delivery within 28 days.']\n",
      "proba: [(1.793810135820966e-64, 0.0), (2.5993503741492003e-61, 0.0), (0.0, 6.63270400397491e-60)]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['ham', 'ham', 'spam']"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "# рандомный объект датасета\n",
    "\n",
    "random_obj_ind = np.random.randint(low=0, high=num_objects, size=3)\n",
    "random_obj_list = df['sms_text'].values[random_obj_ind].tolist()\n",
    "random_target = df['class'][random_obj_ind].tolist()\n",
    "\n",
    "print(random_obj_list)\n",
    "naive_bayes.predict(\n",
    "    random_obj_list\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}