Data_augmentation_Function.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Data_augmentation_Function.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "A8McemKhBPGn"
      },
      "source": [
        "#importing libraries\n",
        "!pip install sentence-transformers\n",
        "!pip install translators\n",
        "!pip install flask-ngrok\n",
        "!pip install flasgger\n",
        "\n",
        "from sentence_transformers import SentenceTransformer\n",
        "model = SentenceTransformer('bert-base-nli-mean-tokens')\n",
        "import spacy\n",
        "nlp = spacy.load('en_core_web_sm')\n",
        "import translators as ts\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import scipy\n",
        "from scipy import spatial\n",
        "from scipy.spatial import distance\n",
        "\n",
        "from flask import Flask, request\n",
        "import flasgger\n",
        "from flasgger import Swagger\n",
        "from flask_ngrok import run_with_ngrok"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e5cxu4QHmviK"
      },
      "source": [
        "#Utility functions\n",
        "def augment(text,lang):\n",
        "  text1 = ts.google(text,from_language = 'en', to_language=lang)\n",
        "  text2 = ts.google(text1,from_language = lang, to_language='en')\n",
        "  return text2\n",
        "\n",
        "#Parameters\n",
        "lang_codes = ['ar','ru','hi','fr','de','it','ja','ko','tr','th']\n",
        "threshold_score = 0.95\n",
        "\n",
        "##Main process function\n",
        "def process(text):\n",
        "  emb1 = model.encode(text, show_progress_bar=False)\n",
        "  ans = []\n",
        "  ans_sent = []\n",
        "  for each in lang_codes:\n",
        "    aug_text = augment(text,each)\n",
        "\n",
        "    #Duplicate checking\n",
        "    if each == aug_text or aug_text in ans_sent:\n",
        "      continue\n",
        "    else:\n",
        "      emb2 = model.encode(aug_text, show_progress_bar=False) \n",
        "      c_s = 1 - spatial.distance.cosine(emb1, emb2)\n",
        "\n",
        "      #Threshold comparison\n",
        "      if c_s > threshold_score and c_s != 1:\n",
        "        #Printing the results\n",
        "        print(aug_text)\n",
        "        ans_sent.append(aug_text)\n",
        "        ans.append([c_s,aug_text])\n",
        "  \n",
        "  #If no augmentation works\n",
        "  if len(ans)==0:\n",
        "    return [\"Sorry no augmented sentences could be generated for the input sentence.\"] \n",
        "  \n",
        "  ans = sorted(ans,reverse = True)\n",
        "  return ans"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vzNj7Loj_hde",
        "outputId": "e9dc57fb-b973-4d96-c984-23763b5e4c48"
      },
      "source": [
        "#Testing the Pipeline\n",
        "text = \"In the end, we all felt like we ate too much.\"\n",
        "a = process(text)\n",
        "print('\\n'.join(map(str, a)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "In the end, we all felt that we ate too much.\n",
            "Finally, we all felt like we ate too much.\n",
            "In the end, we all felt like we eat too much.\n",
            "In the end we all felt, as if we had eaten too much.\n",
            "Finally, we felt that we all have been eating too much.\n",
            "In the end, we all feel like we eat too much.\n",
            "[0.9937810897827148, 'In the end, we all felt like we eat too much.']\n",
            "[0.9931422472000122, 'In the end, we all felt that we ate too much.']\n",
            "[0.9800915718078613, 'In the end we all felt, as if we had eaten too much.']\n",
            "[0.9793198704719543, 'In the end, we all feel like we eat too much.']\n",
            "[0.9684463739395142, 'Finally, we felt that we all have been eating too much.']\n",
            "[0.9646124839782715, 'Finally, we all felt like we ate too much.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "b-RkpNsdiJLy"
      },
      "source": [
        "## Flask App\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NbRNhNc_44iF"
      },
      "source": [
        "app = Flask(__name__)\n",
        "run_with_ngrok(app) \n",
        "Swagger(app)\n",
        "\n",
        "\n",
        "@app.route('/')\n",
        "def welcome():\n",
        "    msg = \"Welcome Everyone. Please add '/apidocs' at the end of the URL for a proper UI.\"\n",
        "    return msg\n",
        "\n",
        "\n",
        "@app.route('/augment', methods=[\"Get\"]) \n",
        "def augment_the_sentence():\n",
        "    \"\"\" Let's augment some sentences\n",
        "    ---\n",
        "    tags:\n",
        "      - Automated Sentence Augmenter\n",
        "    parameters:  \n",
        "      - name: text\n",
        "        in: query\n",
        "        type: string\n",
        "        required: true\n",
        "        description: Type a sentence here\n",
        "    responses:\n",
        "        200:\n",
        "          description: A List of potential augmented sentences \n",
        "        500:\n",
        "          description: The input sentence was not processed properly\n",
        "     \n",
        "    \"\"\"\n",
        "    input_sentence = request.args.get(\"text\")\n",
        "    print(input_sentence)\n",
        "    prediction = process(str(input_sentence))\n",
        "    x = '\\n'.join(map(str, prediction))\n",
        "    print(x)\n",
        "    return x"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "COZ8h6Z9tt9L",
        "outputId": "356d4980-4f50-4d00-c689-e3d3748b10ec"
      },
      "source": [
        "if __name__  == '__main__':\n",
        "  app.run()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            " * Serving Flask app \"__main__\" (lazy loading)\n",
            " * Environment: production\n",
            "\u001b[31m   WARNING: This is a development server. Do not use it in a production deployment.\u001b[0m\n",
            "\u001b[2m   Use a production WSGI server instead.\u001b[0m\n",
            " * Debug mode: off\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            " * Running on http://127.0.0.1:5000/ (Press CTRL+C to quit)\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            " * Running on http://928f-35-185-34-187.ngrok.io\n",
            " * Traffic stats available on http://127.0.0.1:4040\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "127.0.0.1 - - [26/Aug/2021 10:23:39] \"\u001b[37mGET / HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:39] \"\u001b[33mGET /favicon.ico HTTP/1.1\u001b[0m\" 404 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:45] \"\u001b[32mGET /apidocs HTTP/1.1\u001b[0m\" 308 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:46] \"\u001b[37mGET /apidocs/ HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:46] \"\u001b[37mGET /flasgger_static/swagger-ui-bundle.js HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:46] \"\u001b[37mGET /flasgger_static/swagger-ui.css HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:46] \"\u001b[37mGET /flasgger_static/lib/jquery.min.js HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:46] \"\u001b[37mGET /flasgger_static/swagger-ui-standalone-preset.js HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:47] \"\u001b[37mGET /flasgger_static/favicon-32x32.png HTTP/1.1\u001b[0m\" 200 -\n",
            "127.0.0.1 - - [26/Aug/2021 10:23:47] \"\u001b[37mGET /apispec_1.json HTTP/1.1\u001b[0m\" 200 -\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "In the end, we all felt like we ate too much.\n",
            "In the end, we all felt that we ate too much.\n",
            "Finally, we all felt like we ate too much.\n",
            "In the end, we all felt like we eat too much.\n",
            "In the end we all felt, as if we had eaten too much.\n",
            "Finally, we felt that we all have been eating too much.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "127.0.0.1 - - [26/Aug/2021 10:24:10] \"\u001b[37mGET /augment?text=In%20the%20end%2C%20we%20all%20felt%20like%20we%20ate%20too%20much. HTTP/1.1\u001b[0m\" 200 -\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "In the end, we all feel like we eat too much.\n",
            "[0.9937810897827148, 'In the end, we all felt like we eat too much.']\n",
            "[0.9931422472000122, 'In the end, we all felt that we ate too much.']\n",
            "[0.9800915718078613, 'In the end we all felt, as if we had eaten too much.']\n",
            "[0.9793198704719543, 'In the end, we all feel like we eat too much.']\n",
            "[0.9684463739395142, 'Finally, we felt that we all have been eating too much.']\n",
            "[0.9646124839782715, 'Finally, we all felt like we ate too much.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R7KDmKYr8QNK"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}