summarization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "#importing libraries\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "import bs4 as BeautifulSoup\n",
    "import urllib.request  \n",
    "\n",
    "#fetching the content from the URL\n",
    "#fetched_data = urllib.request.urlopen('https://blog.floydhub.com/introduction-to-anomaly-detection-in-python.html')\n",
    "#fetched_data = \"Well, this should have been named \\\"Game of Starks\\\". Would have been more apt, they way the camera followed the Stark siblings only till the end!! Most disappointing end for a promising series i guess. I cant find all the adjectives to describe my angst. This series end could be the most boring, uninteresting, dry, sad, cruel, .... ever in TV history. Even mr. Martin says he is disappointed, but at least they provided a closure to series even if poor. Mr.Martin is never going to finish the books, as he got all his fingers into TV series pie. Easy and quick money! So, the fans have to make their good ending for the series in imadination and be content! I dont know why this series could not have a better happy ending even if spoils Mr.Martin's ego! As he is more into killing off characters. Screen writers to match or outdo mr.Martin morelike and messed it up!\"\n",
    "#article_read = fetched_data.read()\n",
    "\n",
    "#parsing the URL content and storing in a variable\n",
    "#article_parsed = BeautifulSoup.BeautifulSoup(article_read,'html.parser')\n",
    "\n",
    "#returning <p> tags\n",
    "#paragraphs = article_parsed.find_all('p')\n",
    "#paragraphs = \"Well, this should have been named \\\"Game of Starks\\\". Would have been more apt, they way the camera followed the Stark siblings only till the end!! Most disappointing end for a promising series i guess. I cant find all the adjectives to describe my angst. This series end could be the most boring, uninteresting, dry, sad, cruel, .... ever in TV history. Even mr. Martin says he is disappointed, but at least they provided a closure to series even if poor. Mr.Martin is never going to finish the books, as he got all his fingers into TV series pie. Easy and quick money! So, the fans have to make their good ending for the series in imadination and be content! I dont know why this series could not have a better happy ending even if spoils Mr.Martin's ego! As he is more into killing off characters. Screen writers to match or outdo mr.Martin morelike and messed it up!\"\n",
    "article_content = ''\n",
    "\n",
    "#looping through the paragraphs and adding them to the variable\n",
    "#for p in paragraphs:  \n",
    "#    article_content += p.text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _create_dictionary_table(text_string) -> dict:\n",
    "   \n",
    "    #removing stop words\n",
    "    stop_words = set(stopwords.words(\"english\"))\n",
    "    \n",
    "    words = word_tokenize(text_string)\n",
    "    \n",
    "    #reducing words to their root form\n",
    "    stem = PorterStemmer()\n",
    "    \n",
    "    #creating dictionary for the word frequency table\n",
    "    frequency_table = dict()\n",
    "    for wd in words:\n",
    "        wd = stem.stem(wd)\n",
    "        if wd in stop_words:\n",
    "            continue\n",
    "        if wd in frequency_table:\n",
    "            frequency_table[wd] += 1\n",
    "        else:\n",
    "            frequency_table[wd] = 1\n",
    "\n",
    "    return frequency_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _calculate_sentence_scores(sentences, frequency_table) -> dict:   \n",
    "\n",
    "    #algorithm for scoring a sentence by its words\n",
    "    sentence_weight = dict()\n",
    "\n",
    "    for sentence in sentences:\n",
    "        sentence_wordcount = (len(word_tokenize(sentence)))\n",
    "        sentence_wordcount_without_stop_words = 0\n",
    "        for word_weight in frequency_table:\n",
    "            if word_weight in sentence.lower():\n",
    "                sentence_wordcount_without_stop_words += 1\n",
    "                if sentence[:7] in sentence_weight:\n",
    "                    sentence_weight[sentence[:7]] += frequency_table[word_weight]\n",
    "                else:\n",
    "                    sentence_weight[sentence[:7]] = frequency_table[word_weight]\n",
    "\n",
    "        sentence_weight[sentence[:7]] = sentence_weight[sentence[:7]] / sentence_wordcount_without_stop_words\n",
    "\n",
    "       \n",
    "\n",
    "    return sentence_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _calculate_average_score(sentence_weight) -> int:\n",
    "   \n",
    "    #calculating the average score for the sentences\n",
    "    sum_values = 0\n",
    "    for entry in sentence_weight:\n",
    "        sum_values += sentence_weight[entry]\n",
    "\n",
    "    #getting sentence average value from source text\n",
    "    average_score = (sum_values / len(sentence_weight))\n",
    "\n",
    "    return average_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _get_article_summary(sentences, sentence_weight, threshold):\n",
    "    sentence_counter = 0\n",
    "    article_summary = ''\n",
    "\n",
    "    for sentence in sentences:\n",
    "        if sentence[:7] in sentence_weight and sentence_weight[sentence[:7]] >= (threshold):\n",
    "            article_summary += \" \" + sentence\n",
    "            sentence_counter += 1\n",
    "\n",
    "    return article_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _run_article_summary(article):\n",
    "    \n",
    "    #creating a dictionary for the word frequency table\n",
    "    frequency_table = _create_dictionary_table(article)\n",
    "\n",
    "    #tokenizing the sentences\n",
    "    sentences = sent_tokenize(article)\n",
    "\n",
    "    #algorithm for scoring a sentence by its words\n",
    "    sentence_scores = _calculate_sentence_scores(sentences, frequency_table)\n",
    "\n",
    "    #getting the threshold\n",
    "    threshold = _calculate_average_score(sentence_scores)\n",
    "\n",
    "    #producing the summary\n",
    "    article_summary = _get_article_summary(sentences, sentence_scores, 1.5 * threshold)\n",
    "\n",
    "    return article_summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    summary_results = _run_article_summary(article_content)\n",
    "    print(summary_results)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}