MIA_Keyword_Classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MIA Keyword Classification.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/BritneyMuller/colab-notebooks/blob/master/MIA_Keyword_Classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dGjPKxtUngZ5"
      },
      "source": [
        "import sys\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "from textblob import TextBlob"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HK5TvQjannl6"
      },
      "source": [
        "def get_categories(searched_nouns, keywords, n_categories=3):\n",
        "    \"\"\"\n",
        "    return n highest weighted keywords from search terms as tuple\n",
        "    \"\"\"\n",
        "\n",
        "    matches = []\n",
        "\n",
        "    for keyword in keywords.itertuples():\n",
        "        if len(matches) == n_categories:\n",
        "            break\n",
        "\n",
        "        try:\n",
        "            if keyword.Index in searched_nouns:\n",
        "                matches.append(keyword.Index)\n",
        "        except:\n",
        "            continue\n",
        "\n",
        "    missing_count = abs(len(matches) - n_categories)\n",
        "\n",
        "    for _ in range(missing_count):\n",
        "        matches.append(None)\n",
        "\n",
        "    return matches"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "W9bv1I4lnuyJ"
      },
      "source": [
        "def get_nouns(search_terms):\n",
        "    \"\"\"\n",
        "    nltk blob tags\n",
        "    NN: Noun, singular or mass\n",
        "    NNS: Noun, plural\n",
        "    NNP: Proper noun, singular Phrase\n",
        "    NNPS: Proper noun, plural\n",
        "    \"\"\"\n",
        "\n",
        "    nouns = None\n",
        "    try:\n",
        "        blob = TextBlob(search_terms)\n",
        "        nouns = [\n",
        "            noun[0]\n",
        "            for noun in blob.tags\n",
        "            if noun[1] in (\"NN\", \"NNS\", \"NNP\", \"NNPS\")\n",
        "        ]\n",
        "    except:\n",
        "        pass\n",
        "\n",
        "    return nouns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3sKnB6HQnx_5"
      },
      "source": [
        "def get_nouns_phrases(search_terms):\n",
        "    nouns = None\n",
        "    try:\n",
        "        nouns = TextBlob(search_terms).noun_phrases[0]\n",
        "        nouns = tuple(nouns.split())\n",
        "    except:\n",
        "        pass\n",
        "\n",
        "    return nouns"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9Th-LLI5n3M9"
      },
      "source": [
        "def main(input_file, output_file):\n",
        "    df = pd.read_excel(input_file)\n",
        "\n",
        "    # clean column names\n",
        "    df.columns = df.columns.str.lower().str.strip().str.replace(\" \", \"_\")\n",
        "\n",
        "    # df homogenize keywords column - currently mixed types\n",
        "    # make all strings\n",
        "    df.keywords = df.keywords.astype(str)\n",
        "\n",
        "    # row wise -- if no nouns it will be empty\n",
        "    df[\"nouns\"] = df.apply(lambda row: get_nouns(row.keywords), axis=1)\n",
        "\n",
        "    # duplicate record per noun for frequency count\n",
        "    df_exploded = df.explode(\"nouns\")\n",
        "\n",
        "    # TODO: before applying weights, map different keywords that mean the same thing to the unifying keyword\n",
        "    # Highest to lowest\n",
        "    weighted_keywords = df_exploded.groupby(\n",
        "        [\"nouns\"]\n",
        "    ).keywords.agg([\"count\"]).sort_values(by=[\"count\"], ascending=False)\n",
        "\n",
        "    # df['cat_1'], df['cat_2'], df['cat_3'] = [None, None, None]\n",
        "    # df['cat_1'], df['cat_2'], df['cat_3'] = df.apply(\n",
        "    #     # lambda row, weighted_keywords=weighted_keywords: get_categories(row.keywords, weighted_keywords),\n",
        "    #     lambda row: get_categories(row.keywords, weighted_keywords),\n",
        "    #     axis=1\n",
        "    # )\n",
        "    df['results'] = df.apply(\n",
        "        # lambda row, weighted_keywords=weighted_keywords: get_categories(row.keywords, weighted_keywords),\n",
        "        lambda row: get_categories(row.nouns, weighted_keywords),\n",
        "        axis=1\n",
        "    )\n",
        "\n",
        "    df[\n",
        "        ['cat_1', 'cat_2', 'cat_3']\n",
        "    ] = pd.DataFrame(df.results.values.tolist(), index=df.index)\n",
        "\n",
        "\n",
        "    df.to_excel(output_file)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NYVjvYSxuau3"
      },
      "source": [
        "\n",
        "df = pd.read_excel(\"/content/test.xlsx\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4vQyG-FdwSLP",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7e865055-79e8-4f48-9042-6b0eb100cb69"
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "nltk.download('averaged_perceptron_tagger')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package averaged_perceptron_tagger to\n",
            "[nltk_data]     /root/nltk_data...\n",
            "[nltk_data]   Unzipping taggers/averaged_perceptron_tagger.zip.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PTTXZNbsubvT",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "2610cb37-ee82-40b4-ca9c-0b0ff4d211a7"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Keywords</th>\n",
              "      <th>Volume</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>aeropress coffee vs moka pot</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aeropress espresso grind</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>aeropress grind</td>\n",
              "      <td>390.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>aeropress grind level</td>\n",
              "      <td>10.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>aeropress grind setting</td>\n",
              "      <td>20.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                       Keywords  Volume\n",
              "0  aeropress coffee vs moka pot    10.0\n",
              "1      aeropress espresso grind    10.0\n",
              "2               aeropress grind   390.0\n",
              "3         aeropress grind level    10.0\n",
              "4       aeropress grind setting    20.0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-1kTPUAYuipW"
      },
      "source": [
        "# clean column names\n",
        "df.columns = df.columns.str.lower().str.strip().str.replace(\" \", \"_\")\n",
        "\n",
        "# df homogenize keywords column - currently mixed types\n",
        "# make all strings\n",
        "df.keywords = df.keywords.astype(str)\n",
        "\n",
        "# row wise -- if no nouns it will be empty\n",
        "df[\"nouns\"] = df.apply(lambda row: get_nouns(row.keywords), axis=1)\n",
        "\n",
        "# duplicate record per noun for frequency count\n",
        "df_exploded = df.explode(\"nouns\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pk-pX2t4tnJJ"
      },
      "source": [
        "# TODO: before applying weights, map different keywords that mean the same thing to the unifying keyword\n",
        "# Highest to lowest\n",
        "weighted_keywords = df_exploded.groupby(\n",
        "[\"nouns\"]\n",
        ").keywords.agg([\"count\"]).sort_values(by=[\"count\"], ascending=False)\n",
        "\n",
        "# df['cat_1'], df['cat_2'], df['cat_3'] = [None, None, None]\n",
        "# df['cat_1'], df['cat_2'], df['cat_3'] = df.apply(\n",
        "#     # lambda row, weighted_keywords=weighted_keywords: get_categories(row.keywords, weighted_keywords),\n",
        "#     lambda row: get_categories(row.keywords, weighted_keywords),\n",
        "#     axis=1\n",
        "# )\n",
        "df['results'] = df.apply(\n",
        "# lambda row, weighted_keywords=weighted_keywords: get_categories(row.keywords, weighted_keywords),\n",
        "lambda row: get_categories(row.nouns, weighted_keywords),\n",
        "axis=1\n",
        ")\n",
        "\n",
        "df[\n",
        "['cat_1', 'cat_2', 'cat_3']\n",
        "] = pd.DataFrame(df.results.values.tolist(), index=df.index)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KMjtyMv8t0J_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "d20f02e2-3d30-4951-e2a7-48ffcf41fe0b"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>keywords</th>\n",
              "      <th>volume</th>\n",
              "      <th>nouns</th>\n",
              "      <th>results</th>\n",
              "      <th>cat_1</th>\n",
              "      <th>cat_2</th>\n",
              "      <th>cat_3</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>aeropress coffee vs moka pot</td>\n",
              "      <td>10.0</td>\n",
              "      <td>[coffee, vs, moka, pot]</td>\n",
              "      <td>[coffee, moka, pot]</td>\n",
              "      <td>coffee</td>\n",
              "      <td>moka</td>\n",
              "      <td>pot</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>aeropress espresso grind</td>\n",
              "      <td>10.0</td>\n",
              "      <td>[aeropress, espresso, grind]</td>\n",
              "      <td>[espresso, grind, aeropress]</td>\n",
              "      <td>espresso</td>\n",
              "      <td>grind</td>\n",
              "      <td>aeropress</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>aeropress grind</td>\n",
              "      <td>390.0</td>\n",
              "      <td>[aeropress, grind]</td>\n",
              "      <td>[grind, aeropress, None]</td>\n",
              "      <td>grind</td>\n",
              "      <td>aeropress</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>aeropress grind level</td>\n",
              "      <td>10.0</td>\n",
              "      <td>[aeropress, grind, level]</td>\n",
              "      <td>[grind, aeropress, level]</td>\n",
              "      <td>grind</td>\n",
              "      <td>aeropress</td>\n",
              "      <td>level</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>aeropress grind setting</td>\n",
              "      <td>20.0</td>\n",
              "      <td>[aeropress, grind]</td>\n",
              "      <td>[grind, aeropress, None]</td>\n",
              "      <td>grind</td>\n",
              "      <td>aeropress</td>\n",
              "      <td>None</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                       keywords  volume  ...      cat_2      cat_3\n",
              "0  aeropress coffee vs moka pot    10.0  ...       moka        pot\n",
              "1      aeropress espresso grind    10.0  ...      grind  aeropress\n",
              "2               aeropress grind   390.0  ...  aeropress       None\n",
              "3         aeropress grind level    10.0  ...  aeropress      level\n",
              "4       aeropress grind setting    20.0  ...  aeropress       None\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SXqium-5w2pv"
      },
      "source": [
        "df.to_csv(\"test_output.csv\",index  = False)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ud_MbMdvxD7c"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}