Day 47 - 50 Days of Data Analysis .ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "df3cdff6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The AI revolution is, here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seaborn and matploblib are visualization tools.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pandas is a powerful% data analysis library.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data analysis is a crucial part of machine lea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Python is a popular programming language for d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0                        The AI revolution is, here.\n",
       "1    seaborn and matploblib are visualization tools.\n",
       "2       Pandas is a powerful% data analysis library.\n",
       "3  Data analysis is a crucial part of machine lea...\n",
       "4  Python is a popular programming language for d..."
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"Text_data.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "251a2e0c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Checking for duplicates in text column\n",
    "df.duplicated(subset=['text']).any()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b7bf9fe",
   "metadata": {},
   "source": [
    "## 2. \n",
    "\n",
    "#### Create a copy of the DataFrame and convert the text in the \"text\" column to lowercase letters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "9c8bc3fb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the ai revolution is, here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seaborn and matploblib are visualization tools.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pandas is a powerful% data analysis library.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data analysis is a crucial part of machine lea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python is a popular programming language for d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0                        the ai revolution is, here.\n",
       "1    seaborn and matploblib are visualization tools.\n",
       "2       pandas is a powerful% data analysis library.\n",
       "3  data analysis is a crucial part of machine lea...\n",
       "4  python is a popular programming language for d..."
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a copy of the DataFrame\n",
    "df_copy = df.copy()\n",
    "\n",
    "df_copy['text'] = df_copy['text'].str.lower()\n",
    "df_copy.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5f18a0e",
   "metadata": {},
   "source": [
    "## 3. \n",
    "\n",
    "#### Any character that is not a number or in the alphabet is considered to be a special character. Remove special characters in the text column."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6de37a5f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the ai revolution is here</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seaborn and matploblib are visualization tools</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pandas is a powerful data analysis library</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data analysis is a crucial part of machine lea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python is a popular programming language for d...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0                          the ai revolution is here\n",
       "1     seaborn and matploblib are visualization tools\n",
       "2         pandas is a powerful data analysis library\n",
       "3  data analysis is a crucial part of machine lea...\n",
       "4  python is a popular programming language for d..."
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "df_copy['text'] = df_copy['text'].str.translate(str.maketrans('', '', string.punctuation))\n",
    "df_copy.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67aca5c7",
   "metadata": {},
   "source": [
    "## 4. \n",
    "\n",
    "#### Write a code that adds a column of the length of each row in the text column. Return the row with longest text."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e4815785",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>the ai revolution is here</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seaborn and matploblib are visualization tools</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>pandas is a powerful data analysis library</td>\n",
       "      <td>42</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>data analysis is a crucial part of machine lea...</td>\n",
       "      <td>51</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>python is a popular programming language for d...</td>\n",
       "      <td>57</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  text length\n",
       "0                          the ai revolution is here           25\n",
       "1     seaborn and matploblib are visualization tools           46\n",
       "2         pandas is a powerful data analysis library           42\n",
       "3  data analysis is a crucial part of machine lea...           51\n",
       "4  python is a popular programming language for d...           57"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a text length column\n",
    "df_copy[\"text length\"] = df_copy[\"text\"].str.len()\n",
    "df_copy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "bb530df4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>text length</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>big data is becoming increasingly important in...</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text  text length\n",
       "8  big data is becoming increasingly important in...           59"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Sorting the DataFrame and returning the first row\n",
    "df_copy.sort_values(by = [\"text length\"], ascending=False).iloc[:1]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bb83a5de",
   "metadata": {},
   "source": [
    "## 5\n",
    "\n",
    "#### Stopwords are common words that are typically removed from text data before performing natural language processing tasks, such as text classification, sentiment analysis, or information retrieval. Examples of stopwords include \"the\", \"and\", \"a\", \"an\", \"in\", \"of\", \"to\", etc. Write a code to remove stop words in the text using nltk module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "1d2913c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>The AI revolution is, here.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>seaborn matploblib visualization tools.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Pandas powerful% data analysis library.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Data analysis crucial part machine learning.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Python popular programming language data science.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text\n",
       "0                        The AI revolution is, here.\n",
       "1            seaborn matploblib visualization tools.\n",
       "2            Pandas powerful% data analysis library.\n",
       "3       Data analysis crucial part machine learning.\n",
       "4  Python popular programming language data science."
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "\n",
    "# Accessing 'English' stopwords\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "\n",
    "# Returning words that are not in stopwords\n",
    "df['text'] = df['text'].apply(lambda x: ' '.join([word for word in x.split() \n",
    "                                                  if word not in stop_words]))\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a25ed0d",
   "metadata": {},
   "source": [
    "## 6.\t\n",
    "\n",
    "#### For text data to be used in machine learning, it must be transformed into numeric vectors. Using scikit-learn, tokenize the text into numeric vectors.\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e963b720",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ai</th>\n",
       "      <th>analysis</th>\n",
       "      <th>becoming</th>\n",
       "      <th>big</th>\n",
       "      <th>changing</th>\n",
       "      <th>crucial</th>\n",
       "      <th>data</th>\n",
       "      <th>here</th>\n",
       "      <th>important</th>\n",
       "      <th>increasingly</th>\n",
       "      <th>...</th>\n",
       "      <th>science</th>\n",
       "      <th>seaborn</th>\n",
       "      <th>statistics</th>\n",
       "      <th>subfield</th>\n",
       "      <th>the</th>\n",
       "      <th>today</th>\n",
       "      <th>tools</th>\n",
       "      <th>transforming</th>\n",
       "      <th>visualization</th>\n",
       "      <th>world</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   ai  analysis  becoming  big  changing  crucial  data  here  important   \n",
       "0   1         0         0    0         0        0     0     1          0  \\\n",
       "1   0         0         0    0         0        0     0     0          0   \n",
       "2   0         1         0    0         0        0     1     0          0   \n",
       "3   0         1         0    0         0        1     1     0          0   \n",
       "4   0         0         0    0         0        0     1     0          0   \n",
       "\n",
       "   increasingly  ...  science  seaborn  statistics  subfield  the  today   \n",
       "0             0  ...        0        0           0         0    1      0  \\\n",
       "1             0  ...        0        1           0         0    0      0   \n",
       "2             0  ...        0        0           0         0    0      0   \n",
       "3             0  ...        0        0           0         0    0      0   \n",
       "4             0  ...        1        0           0         0    0      0   \n",
       "\n",
       "   tools  transforming  visualization  world  \n",
       "0      0             0              0      0  \n",
       "1      1             0              1      0  \n",
       "2      0             0              0      0  \n",
       "3      0             0              0      0  \n",
       "4      0             0              0      0  \n",
       "\n",
       "[5 rows x 42 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "\n",
    "# Instantianting the Vectorizer\n",
    "cv = CountVectorizer()\n",
    "\n",
    "# Fitting the text\n",
    "word_count_matrix = cv.fit_transform(df.text)\n",
    "\n",
    "# Returning a DataFrame of vectorized text\n",
    "word_count_df = pd.DataFrame(word_count_matrix.toarray(), columns=cv.get_feature_names_out())\n",
    "word_count_df.head() "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1e934fd",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}