ARIAM_MODEL_testing_code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea9815ab-47b7-4f27-91f3-374dd576ee7e",
   "metadata": {},
   "outputs": [],
   "source": [
    "Code/ Chunk:1\n",
    "\n",
    "Data Preprocessing: \n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import yfinance as yf\n",
    "import talib\n",
    "import logging\n",
    "import plotly.io as pio\n",
    "import plotly.offline as py\n",
    "import plotly.graph_objs as go\n",
    "from IPython.display import display, HTML\n",
    "from plotly.subplots import make_subplots\n",
    "\n",
    "\n",
    "# Setting up logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message=s')\n",
    "\n",
    "def download_stock_data(ticker: str, start_date: str, end_date: str) -> pd.DataFrame:\n",
    "    \"\"\"Downloads stock data from Yahoo Finance.\"\"\"\n",
    "    try:\n",
    "        logging.info(f\"Downloading stock data for ticker: {ticker}\")\n",
    "        data = yf.download(ticker, start=start_date, end=end_date)\n",
    "        data.reset_index(inplace=True)\n",
    "        data['Date'] = pd.to_datetime(data['Date'])\n",
    "        data.set_index('Date', inplace=True)\n",
    "        return data\n",
    "    except Exception as e:\n",
    "        logging.error(f\"Failed to download stock data: {e}\")\n",
    "        raise\n",
    "\n",
    "def preprocess_data(data: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"Preprocesses the stock data by adding technical indicators.\"\"\"\n",
    "    logging.info(\"Starting preprocessing of data\")\n",
    "    \n",
    "    # Adding technical indicators without normalizing\n",
    "    data['SMA_10'] = talib.SMA(data['Close'], timeperiod=10)\n",
    "    data['EMA_10'] = talib.EMA(data['Close'], timeperiod=10)\n",
    "    data['RSI'] = talib.RSI(data['Close'], timeperiod=14)\n",
    "    data['MACD'], data['MACD_signal'], _ = talib.MACD(data['Close'])\n",
    "    data.fillna(method='bfill', inplace=True)\n",
    "    logging.info(\"Data preprocessing complete\")\n",
    "    return data\n",
    "\n",
    "def main() -> pd.DataFrame:\n",
    "    ticker_symbol = 'TATAELXSI.NS'\n",
    "    start_date = '2019-04-01'\n",
    "    end_date = '2024-03-31'\n",
    "    stock_data = download_stock_data(ticker_symbol, start_date, end_date)\n",
    "    preprocessed_data = preprocess_data(stock_data)\n",
    "    return preprocessed_data\n",
    "\n",
    "# Run main and get the preprocessed data\n",
    "preprocessed_data = main()\n",
    "\n",
    "# Displaying the first 5 rows with a title\n",
    "display(HTML('<h2>First 5 Rows of the DataFrame</h2>'))\n",
    "display(HTML(preprocessed_data.head(5).to_html()))\n",
    "# Displaying the last 5 rows with a title\n",
    "display(HTML('<h2>Last 5 Rows of the DataFrame</h2>'))\n",
    "display(HTML(preprocessed_data.tail(5).to_html()))\n",
    "\n",
    "\n",
    "Code/Chunk:2\n",
    "\n",
    "Data aggregation\n",
    "\n",
    "def aggregate_to_monthly_average(data: pd.DataFrame) -> pd.DataFrame:\n",
    "    \"\"\"Aggregates daily data into monthly averages.\"\"\"\n",
    "    try:\n",
    "        logging.info(\"Starting the aggregation of data to monthly averages.\")\n",
    "        monthly_data_avg = data.resample('M').mean()\n",
    "        logging.info(\"Data successfully aggregated to monthly averages.\")\n",
    "        return monthly_data_avg\n",
    "    except Exception as e:\n",
    "        logging.error(f\"Failed to aggregate data: {e}\")\n",
    "        raise\n",
    "\n",
    "def plot_trend_comparison(daily_data: pd.DataFrame, monthly_data: pd.DataFrame, title=\"Daily vs. Monthly Trend Comparison\") -> None:\n",
    "    \"\"\"Plots the daily trend and monthly average trend side-by-side using Plotly.\"\"\"\n",
    "    try:\n",
    "        logging.info(\"Starting to plot the trend comparison.\")\n",
    "        fig = make_subplots(rows=1, cols=2, subplot_titles=('Daily Trend', 'Monthly Average Trend'))\n",
    "        \n",
    "        # Daily plot\n",
    "        daily_trace = go.Scatter(x=daily_data.index, y=daily_data['Close'], mode='lines', name='Daily Close', line=dict(color='blue'))\n",
    "        fig.add_trace(daily_trace, row=1, col=1)\n",
    "        \n",
    "        # Monthly plot\n",
    "        monthly_trace = go.Scatter(x=monthly_data.index, y=monthly_data['Close'], mode='lines', name='Monthly Close', line=dict(color='red'))\n",
    "        fig.add_trace(monthly_trace, row=1, col=2)\n",
    "        \n",
    "        # Layout\n",
    "        fig.update_layout(title_text=title, hovermode='x', showlegend=False)\n",
    "        \n",
    "        pio.show(fig)  # Display the plot inline\n",
    "        logging.info(\"Successfully plotted the trend comparison.\")\n",
    "    except Exception as e:\n",
    "        logging.error(f\"Failed to plot the trend comparison: {e}\")\n",
    "        raise\n",
    "\n",
    "# Aggregate and plot the data\n",
    "monthly_data = aggregate_to_monthly_average(preprocessed_data)\n",
    "plot_trend_comparison(preprocessed_data, monthly_data)\n",
    "\n",
    "\n",
    "#print(monthly_data.head(5))\n",
    "# Displaying the last 5 rows with a title\n",
    "display(HTML('<h2>Last 5 Rows of the DataFrame</h2>'))\n",
    "display(HTML(monthly_data.tail(5).to_html()))\n",
    "\n",
    "Code/Chunk:3\n",
    "convert to stationary to implement model\n",
    "\n",
    "from statsmodels.tsa.stattools import adfuller\n",
    "\n",
    "def test_stationarity(timeseries: pd.Series) -> None:\n",
    "    \"\"\"Tests and logs the stationarity of the provided timeseries.\"\"\"\n",
    "    logging.info(\"Testing the stationarity of the timeseries\")\n",
    "    dftest = adfuller(timeseries, autolag='AIC')\n",
    "    dfoutput = pd.Series(dftest[:4], index=['Test Statistic','p-value','#Lags Used','Number of Observations Used'])\n",
    "    for key,value in dftest[4].items():\n",
    "        dfoutput['Critical Value (%s)' % key] = value\n",
    "    logging.info(\"Results of Dickey-Fuller Test:\")\n",
    "    logging.info(dfoutput)\n",
    "\n",
    "def make_stationary(data: pd.DataFrame, target_column: str) -> pd.DataFrame:\n",
    "    \"\"\"Transforms the target column to make the data stationary.\"\"\"\n",
    "    logging.info(\"Making data stationary\")\n",
    "    \n",
    "    # Apply log transformation\n",
    "    data[f'{target_column}_log'] = np.log(data[target_column])\n",
    "    \n",
    "    # Apply differencing\n",
    "    data[f'{target_column}_stationary'] = data[f'{target_column}_log'].diff().dropna()\n",
    "\n",
    "    # Test stationarity\n",
    "    test_stationarity(data[f'{target_column}_stationary'].dropna())\n",
    "\n",
    "    return data\n",
    "\n",
    "# Example usage\n",
    "def main():\n",
    "    # Assuming preprocessed_data is already loaded from previous steps\n",
    "    target_column = 'Close'  # Define which column to make stationary\n",
    "    preprocessed_data.dropna(inplace=True)  # Drop NaN values for stationarity testing\n",
    "    stationary_data = make_stationary(preprocessed_data, target_column)\n",
    "    \n",
    "    logging.info(\"Stationarity transformation complete\")\n",
    "    return stationary_data\n",
    "\n",
    "# Run main to perform stationarity transformation\n",
    "stationary_data = main()\n",
    "\n",
    "\n",
    "# Print the stationary data\n",
    "#print(stationary_data)\n",
    "\n",
    "# Displaying the last 5 rows with a title\n",
    "display(HTML('<h2>Last 5 Rows of the DataFrame</h2>'))\n",
    "display(HTML(stationary_data.head(5).to_html()))\n",
    "\n",
    "Code/Chunk:5\n",
    "\n",
    "ARIMA model Implementation - Univariate model\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import logging\n",
    "from statsmodels.tsa.statespace.sarimax import SARIMAX\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objs as go\n",
    "from statsmodels.tsa.stattools import acf, pacf\n",
    "from IPython.display import display, HTML\n",
    "\n",
    "# Setting up logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message=s')\n",
    "\n",
    "# Function to split data into training and testing datasets\n",
    "def split_data(data: pd.DataFrame, start_train: str, end_train: str, start_test: str, end_test: str) -> (pd.DataFrame, pd.DataFrame):\n",
    "    \"\"\"Splits the data into training and testing datasets.\"\"\"\n",
    "    train_data = data[start_train:end_train]\n",
    "    test_data = data[start_test:end_test]\n",
    "    return train_data, test_data\n",
    "\n",
    "# Function to implement SARIMAX model and return fitted model\n",
    "def train_sarimax(train_data: pd.Series, p: int, d: int, q: int, seasonal_order=(0, 0, 0, 0)):\n",
    "    \"\"\"Trains the SARIMAX model and returns the fitted model.\"\"\"\n",
    "    model = SARIMAX(train_data, order=(p, d, q), seasonal_order=seasonal_order)\n",
    "    model_fit = model.fit()\n",
    "    return model_fit\n",
    "\n",
    "# Function to visualize the train/test/forecast results with Plotly\n",
    "def plot_train_test_forecast(train_data: pd.Series, test_data: pd.Series, forecast: pd.Series) -> None:\n",
    "    \"\"\"Visualizes the train, test, and forecast data using Plotly.\"\"\"\n",
    "    fig = make_subplots(rows=1, cols=1)\n",
    "    \n",
    "    # Add traces for training, testing, and forecasted data\n",
    "    fig.add_trace(go.Scatter(x=train_data.index, y=train_data, mode='lines', name='Train Data'))\n",
    "    fig.add_trace(go.Scatter(x=test_data.index, y=test_data, mode='lines', name='Test Data'))\n",
    "    fig.add_trace(go.Scatter(x=forecast.index, y=forecast, mode='lines', name='Forecast'))\n",
    "    \n",
    "    # Set plot layout\n",
    "    fig.update_layout(\n",
    "        title='Train, Test and Forecast Data',\n",
    "        xaxis_title='Date',\n",
    "        yaxis_title='Stock Price',\n",
    "        hovermode='x',\n",
    "        showlegend=True\n",
    "    )\n",
    "    \n",
    "    fig.show()\n",
    "\n",
    "# Function to plot ACF and PACF with Plotly\n",
    "def plot_acf_pacf(data: pd.Series) -> None:\n",
    "    \"\"\"Plots the ACF and PACF using Plotly.\"\"\"\n",
    "    acf_values = acf(data, nlags=40)\n",
    "    pacf_values = pacf(data, nlags=40)\n",
    "    \n",
    "    fig = make_subplots(rows=1, cols=2, subplot_titles=('ACF', 'PACF'))\n",
    "\n",
    "    fig.add_trace(go.Bar(x=np.arange(len(acf_values)), y=acf_values, name='ACF'), row=1, col=1)\n",
    "    fig.add_trace(go.Bar(x=np.arange(len(pacf_values)), y=pacf_values, name='PACF'), row=1, col=2)\n",
    "\n",
    "    fig.update_layout(title_text='ACF and PACF Plots', showlegend=False)\n",
    "    fig.show()\n",
    "\n",
    "# Main function for SARIMAX implementation and forecasting\n",
    "def main():\n",
    "    # Assuming `stationary_data` is already loaded from previous steps\n",
    "    target_column = 'Close_stationary'\n",
    "    train_start, train_end = '01-04-2019', '31-03-2023'\n",
    "    test_start, test_end = '01-04-2023', '31-03-2024'\n",
    "    forecast_start, forecast_end = '01-04-2024', '31-03-2025'\n",
    "\n",
    "    train_data, test_data = split_data(stationary_data[target_column].dropna(), train_start, train_end, test_start, test_end)\n",
    "    \n",
    "    # Plot ACF and PACF plots for SARIMAX parameter selection\n",
    "    plot_acf_pacf(train_data)\n",
    "    \n",
    "    # Train SARIMAX model with assumed p, d, q values and seasonal_order (these should be fine-tuned)\n",
    "    p, d, q, seasonal_order = 1, 1, 1, (1, 1, 1, 12)\n",
    "    model_fit = train_sarimax(train_data, p, d, q, seasonal_order)\n",
    "    \n",
    "    # Forecast for the next 12 months\n",
    "    forecast_steps = pd.date_range(start=forecast_start, end=forecast_end, freq='M').size\n",
    "    forecast = model_fit.forecast(steps=forecast_steps)\n",
    "    \n",
    "    # Set the forecast index to the expected date range\n",
    "    forecast.index = pd.date_range(start=forecast_start, end=forecast_end, freq='M')\n",
    "\n",
    "    # Visualization\n",
    "    plot_train_test_forecast(train_data, test_data, forecast)\n",
    "    \n",
    "    # Print SARIMAX results\n",
    "    print(model_fit.summary())\n",
    "\n",
    "# Run the main function\n",
    "main()\n",
    "\n",
    "\n",
    "Code/Chunk:6\n",
    "\n",
    "Analyze Model Performance\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import logging\n",
    "from statsmodels.tsa.statespace.sarimax import SARIMAX\n",
    "from plotly.subplots import make_subplots\n",
    "import plotly.graph_objs as go\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "from statsmodels.tsa.stattools import acf, pacf\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from scipy import stats\n",
    "\n",
    "# Set up logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message=s')\n",
    "\n",
    "# Function to split data into training and testing datasets\n",
    "def split_data(data: pd.DataFrame, start_train: str, end_train: str, start_test: str, end_test: str) -> (pd.DataFrame, pd.DataFrame):\n",
    "    \"\"\"Splits the data into training and testing datasets.\"\"\"\n",
    "    train_data = data[start_train:end_train]\n",
    "    test_data = data[start_test:end_test]\n",
    "    return train_data, test_data\n",
    "\n",
    "# Function to implement SARIMAX model and return fitted model\n",
    "def train_sarimax(train_data: pd.Series, p: int, d: int, q: int, seasonal_order=(0, 0, 0, 0)):\n",
    "    \"\"\"Trains the SARIMAX model and returns the fitted model.\"\"\"\n",
    "    model = SARIMAX(train_data, order=(p, d, q), seasonal_order=seasonal_order)\n",
    "    model_fit = model.fit()\n",
    "    return model_fit\n",
    "\n",
    "# Function to analyze residuals\n",
    "def plot_residuals(residuals):\n",
    "    \"\"\"Analyzes residuals for autocorrelation, normality, and heteroscedasticity.\"\"\"\n",
    "    fig, ax = plt.subplots(1, 3, figsize=(18, 5))\n",
    "    \n",
    "    # Residuals vs Time\n",
    "    sns.lineplot(x=residuals.index, y=residuals, ax=ax[0])\n",
    "    ax[0].set_title('Residuals vs Time')\n",
    "    \n",
    "    # Histogram\n",
    "    sns.histplot(residuals, kde=True, ax=ax[1])\n",
    "    ax[1].set_title('Histogram of Residuals')\n",
    "    \n",
    "    # Q-Q Plot\n",
    "    stats.probplot(residuals, dist=\"norm\", plot=ax[2])\n",
    "    ax[2].set_title('Q-Q Plot')\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "# Function to calculate and print model accuracy metrics\n",
    "def calculate_accuracy(test_data, forecast):\n",
    "    \"\"\"Calculates and prints MSE, MAE, and RMSE.\"\"\"\n",
    "    mse = mean_squared_error(test_data, forecast)\n",
    "    mae = mean_absolute_error(test_data, forecast)\n",
    "    rmse = np.sqrt(mse)\n",
    "    \n",
    "    logging.info(f'MSE: {mse:.3f}')\n",
    "    logging.info(f'MAE: {mae:.3f}')\n",
    "    logging.info(f'RMSE: {rmse:.3f}')\n",
    "    \n",
    "    print(f'MSE: {mse:.3f}')\n",
    "    print(f'MAE: {mae:.3f}')\n",
    "    print(f'RMSE: {rmse:.3f}')\n",
    "\n",
    "# Main function to train and analyze SARIMAX model\n",
    "def main():\n",
    "    # Assuming `stationary_data` is already loaded from previous steps\n",
    "    target_column = 'Close_stationary'\n",
    "    train_start, train_end = '01-04-2019', '31-03-2023'\n",
    "    test_start, test_end = '01-04-2023', '31-03-2024'\n",
    "    \n",
    "    train_data, test_data = split_data(stationary_data[target_column].dropna(), train_start, train_end, test_start, test_end)\n",
    "    \n",
    "    # Train SARIMAX model with assumed p, d, q values and seasonal_order (these should be fine-tuned)\n",
    "    p, d, q, seasonal_order = 1, 1, 1, (1, 1, 1, 12)\n",
    "    model_fit = train_sarimax(train_data, p, d, q, seasonal_order)\n",
    "    \n",
    "    # Forecast for the test period\n",
    "    forecast = model_fit.forecast(steps=len(test_data))\n",
    "    \n",
    "    # Plot residuals\n",
    "    plot_residuals(model_fit.resid)\n",
    "    \n",
    "    # Evaluate accuracy metrics\n",
    "    calculate_accuracy(test_data, forecast)\n",
    "    \n",
    "    # Print SARIMAX summary\n",
    "    print(model_fit.summary())\n",
    "\n",
    "# Run the main function\n",
    "main()\n",
    "\n",
    "\n",
    "Check the code end to end. \n",
    "Code is maintain the code flow?\n",
    "Code is advanced to meet the industrial standards? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "1798a1c2-3e0c-40ad-9c4b-3070df092399",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Looking in links: /usr/share/pip-wheels\n",
      "Requirement already satisfied: yfinance in /home/e1ba06db-f639-4812-9369-20fe39201d9e/.local/lib/python3.10/site-packages (0.2.38)\n",
      "Requirement already satisfied: pandas>=1.3.0 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (2.1.4)\n",
      "Requirement already satisfied: numpy>=1.16.5 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (1.26.4)\n",
      "Requirement already satisfied: requests>=2.31 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (2.31.0)\n",
      "Requirement already satisfied: multitasking>=0.0.7 in /home/e1ba06db-f639-4812-9369-20fe39201d9e/.local/lib/python3.10/site-packages (from yfinance) (0.0.11)\n",
      "Requirement already satisfied: lxml>=4.9.1 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (4.9.3)\n",
      "Requirement already satisfied: appdirs>=1.4.4 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (1.4.4)\n",
      "Requirement already satisfied: pytz>=2022.5 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (2023.3.post1)\n",
      "Requirement already satisfied: frozendict>=2.3.4 in /home/e1ba06db-f639-4812-9369-20fe39201d9e/.local/lib/python3.10/site-packages (from yfinance) (2.4.2)\n",
      "Requirement already satisfied: peewee>=3.16.2 in /home/e1ba06db-f639-4812-9369-20fe39201d9e/.local/lib/python3.10/site-packages (from yfinance) (3.17.3)\n",
      "Requirement already satisfied: beautifulsoup4>=4.11.1 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from yfinance) (4.12.2)\n",
      "Requirement already satisfied: html5lib>=1.1 in /home/e1ba06db-f639-4812-9369-20fe39201d9e/.local/lib/python3.10/site-packages (from yfinance) (1.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from beautifulsoup4>=4.11.1->yfinance) (2.5)\n",
      "Requirement already satisfied: six>=1.9 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from html5lib>=1.1->yfinance) (1.16.0)\n",
      "Requirement already satisfied: webencodings in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from html5lib>=1.1->yfinance) (0.5.1)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from pandas>=1.3.0->yfinance) (2.8.2)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from pandas>=1.3.0->yfinance) (2023.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from requests>=2.31->yfinance) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from requests>=2.31->yfinance) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from requests>=2.31->yfinance) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/envs/anaconda-ai-2024.04-py310/lib/python3.10/site-packages (from requests>=2.31->yfinance) (2024.2.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install yfinance"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8f5906d-0257-4d30-8080-a0d365192e98",
   "metadata": {},
   "source": [
    "# ARIMA MODEL FINAL OUTPUT"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2556e729-aa53-4b45-b433-05d23ef8e14e",
   "metadata": {},
   "source": [
    "### Data Preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "e7605f97-7eec-4aa0-b568-7d32f243eb38",
   "metadata": {},
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'yfinance'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[43], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mpandas\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mpd\u001b[39;00m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m----> 3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01myfinance\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01myf\u001b[39;00m\n\u001b[1;32m      4\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mtalib\u001b[39;00m\n\u001b[1;32m      5\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mlogging\u001b[39;00m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'yfinance'"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import yfinance as yf\n",
    "import talib\n",
    "import logging\n",
    "\n",
    "class DataPipeline:\n",
    "    def __init__(self, ticker: str, start_date: str, end_date: str):\n",
    "        self.ticker = ticker\n",
    "        self.start_date = start_date\n",
    "        self.end_date = end_date\n",
    "        self.data = None\n",
    "    \n",
    "    def download_stock_data(self) -> pd.DataFrame:\n",
    "        \"\"\"Download stock data from Yahoo Finance.\"\"\"\n",
    "        try:\n",
    "            logging.info(f\"Downloading stock data for ticker: {self.ticker}\")\n",
    "            self.data = yf.download(self.ticker, start=self.start_date, end=self.end_date)\n",
    "            self.data.reset_index(inplace=True)\n",
    "            self.data['Date'] = pd.to_datetime(self.data['Date'])\n",
    "            self.data.set_index('Date', inplace=True)\n",
    "            return self.data\n",
    "        except Exception as e:\n",
    "            logging.error(f\"Failed to download stock data: {e}\")\n",
    "            raise\n",
    "    \n",
    "    def preprocess_data(self) -> pd.DataFrame:\n",
    "        \"\"\"Add technical indicators to the stock data.\"\"\"\n",
    "        logging.info(\"Starting preprocessing of data\")\n",
    "        \n",
    "        # Adding technical indicators\n",
    "        self.data['SMA_10'] = talib.SMA(self.data['Close'], timeperiod=10)\n",
    "        self.data['EMA_10'] = talib.EMA(self.data['Close'], timeperiod=10)\n",
    "        self.data['RSI'] = talib.RSI(self.data['Close'], timeperiod=14)\n",
    "        self.data['MACD'], self.data['MACD_signal'], _ = talib.MACD(self.data['Close'])\n",
    "        \n",
    "        # Fill missing values by backfilling\n",
    "        self.data.fillna(method='bfill', inplace=True)\n",
    "        logging.info(\"Data preprocessing complete\")\n",
    "        return self.data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "216a0bbb-0ebd-4f1b-87f9-ae4cf7388d56",
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == \"__main__\":\n",
    "    # Set up logging\n",
    "    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')\n",
    "    \n",
    "    # Initialize DataPipeline\n",
    "    ticker_symbol = 'TATAELXSI.NS'\n",
    "    start_date = '2019-04-01'\n",
    "    end_date = '2024-03-31'\n",
    "    pipeline = DataPipeline(ticker_symbol, start_date, end_date)\n",
    "    \n",
    "    # Download and preprocess data\n",
    "    pipeline.download_stock_data()\n",
    "    preprocessed_data = pipeline.preprocess_data()\n",
    "    \n",
    "    # Displaying the data\n",
    "    display(HTML('<h2>First 5 Rows of the DataFrame</h2>'))\n",
    "    display(HTML(preprocessed_data.head(5).to_html()))\n",
    "    display(HTML('<h2>Last 5 Rows of the DataFrame</h2>'))\n",
    "    display(HTML(preprocessed_data.tail(5).to_html()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3dbd8e94-a597-462d-a7cf-b50bd8348393",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1149bba8-4698-4860-ac9d-e45934d1d1ef",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9bd1b8f-bcad-44ec-bcf7-ceeb62047b0b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f2c061a4-f0ff-47bd-ae14-47b452c056a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import logging\n",
    "from statsmodels.tsa.statespace.sarimax import SARIMAX\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "import plotly.graph_objs as go\n",
    "from plotly.subplots import make_subplots\n",
    "from scipy import stats\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# Set up logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message=s')\n",
    "\n",
    "# Function to split data into training and testing datasets\n",
    "def split_data(data: pd.DataFrame, start_train: str, end_train: str, start_test: str, end_test: str) -> (pd.DataFrame, pd.DataFrame):\n",
    "    \"\"\"Splits the data into training and testing datasets.\"\"\"\n",
    "    train_data = data[start_train:end_train]\n",
    "    test_data = data[start_test:end_test]\n",
    "    return train_data, test_data\n",
    "\n",
    "# Function to train SARIMAX model and return fitted model\n",
    "def train_sarimax(train_data: pd.Series, p: int, d: int, q: int, seasonal_order=(0, 0, 0, 0)):\n",
    "    \"\"\"Trains the SARIMAX model and returns the fitted model.\"\"\"\n",
    "    model = SARIMAX(train_data, order=(p, d, q), seasonal_order=seasonal_order)\n",
    "    model_fit = model.fit()\n",
    "    return model_fit\n",
    "\n",
    "# Function to calculate and print model accuracy metrics\n",
    "def calculate_accuracy(test_data, forecast):\n",
    "    \"\"\"Calculates and prints MSE, MAE, and RMSE.\"\"\"\n",
    "    mse = mean_squared_error(test_data, forecast)\n",
    "    mae = mean_absolute_error(test_data, forecast)\n",
    "    rmse = np.sqrt(mse)\n",
    "    \n",
    "    logging.info(f'MSE: {mse:.3f}')\n",
    "    logging.info(f'MAE: {mae:.3f}')\n",
    "    logging.info(f'RMSE: {rmse:.3f}')\n",
    "    \n",
    "    print(f'MSE: {mse:.3f}')\n",
    "    print(f'MAE: {mae:.3f}')\n",
    "    print(f'RMSE: {rmse:.3f}')\n",
    "\n",
    "# Function to visualize train, test, and forecast data using Plotly\n",
    "def plot_train_test_forecast(train_data: pd.Series, test_data: pd.Series, forecast: pd.Series) -> None:\n",
    "    \"\"\"Visualizes the train, test, and forecast data using Plotly.\"\"\"\n",
    "    fig = make_subplots(rows=1, cols=1)\n",
    "    \n",
    "    # Add traces for training, testing, and forecasted data\n",
    "    fig.add_trace(go.Scatter(x=train_data.index, y=train_data, mode='lines', name='Train Data'))\n",
    "    fig.add_trace(go.Scatter(x=test_data.index, y=test_data, mode='lines', name='Test Data'))\n",
    "    fig.add_trace(go.Scatter(x=forecast.index, y=forecast, mode='lines', name='Forecast'))\n",
    "    \n",
    "    # Set plot layout\n",
    "    fig.update_layout(\n",
    "        title='Train, Test and Forecast Data',\n",
    "        xaxis_title='Date',\n",
    "        yaxis_title='Stock Price',\n",
    "        hovermode='x',\n",
    "        showlegend=True\n",
    "    )\n",
    "    \n",
    "    fig.show()\n",
    "\n",
    "# Main function for final model evaluation, visualization, and reporting\n",
    "def main():\n",
    "    # Assuming `stationary_data` is already loaded from previous steps\n",
    "    target_column = 'Close_stationary'\n",
    "    train_start, train_end = '01-04-2019', '31-03-2023'\n",
    "    test_start, test_end = '01-04-2023', '31-03-2024'\n",
    "    forecast_start, forecast_end = '01-04-2024', '31-03-2025'\n",
    "\n",
    "    train_data, test_data = split_data(stationary_data[target_column].dropna(), train_start, train_end, test_start, test_end)\n",
    "    \n",
    "    # Train SARIMAX model with assumed p, d, q values and seasonal_order\n",
    "    p, d, q, seasonal_order = 1, 1, 1, (1, 1, 1, 12)\n",
    "    model_fit = train_sarimax(train_data, p, d, q, seasonal_order)\n",
    "    \n",
    "    # Forecast for the next 12 months\n",
    "    forecast_steps = pd.date_range(start=forecast_start, end=forecast_end, freq='M').size\n",
    "    forecast = model_fit.forecast(steps=forecast_steps)\n",
    "    \n",
    "    # Set the forecast index to the expected date range\n",
    "    forecast.index = pd.date_range(start=forecast_start, end=forecast_end, freq='M')\n",
    "\n",
    "    # Plot and visualize the forecast along with train and test data\n",
    "    plot_train_test_forecast(train_data, test_data, forecast)\n",
    "    \n",
    "    # Evaluate the model accuracy for the test period\n",
    "    forecast_test = model_fit.forecast(steps=len(test_data))\n",
    "    calculate_accuracy(test_data, forecast_test)\n",
    "    \n",
    "    # Print SARIMAX summary\n",
    "    print(model_fit.summary())\n",
    "\n",
    "# Run the main function\n",
    "main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "91e3b191-31ed-43f7-b887-36ac59317e24",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import logging\n",
    "from statsmodels.tsa.statespace.sarimax import SARIMAX\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error\n",
    "from scipy import stats\n",
    "\n",
    "# Set up logging\n",
    "logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname=s - %(message=s')\n",
    "\n",
    "# Function to split data into training and testing datasets\n",
    "def split_data(data: pd.DataFrame, start_train: str, end_train: str, start_test: str, end_test: str) -> (pd.DataFrame, pd.DataFrame):\n",
    "    \"\"\"Splits the data into training and testing datasets.\"\"\"\n",
    "    train_data = data[start_train:end_train]\n",
    "    test_data = data[start_test:end_test]\n",
    "    return train_data, test_data\n",
    "\n",
    "# Function to implement SARIMAX model and return fitted model\n",
    "def train_sarimax(train_data: pd.Series, p: int, d: int, q: int, seasonal_order=(0, 0, 0, 0)):\n",
    "    \"\"\"Trains the SARIMAX model and returns the fitted model.\"\"\"\n",
    "    model = SARIMAX(train_data, order=(p, d, q), seasonal_order=seasonal_order)\n",
    "    model_fit = model.fit()\n",
    "    return model_fit\n",
    "\n",
    "# Function to analyze residuals\n",
    "def plot_residuals(residuals):\n",
    "    \"\"\"Analyzes residuals for autocorrelation, normality, and heteroscedasticity.\"\"\"\n",
    "    fig, ax = plt.subplots(1, 3, figsize=(18, 5))\n",
    "    \n",
    "    # Residuals vs Time\n",
    "    sns.lineplot(x=residuals.index, y=residuals, ax=ax[0])\n",
    "    ax[0].set_title('Residuals vs Time')\n",
    "    \n",
    "    # Histogram\n",
    "    sns.histplot(residuals, kde=True, ax=ax[1])\n",
    "    ax[1].set_title('Histogram of Residuals')\n",
    "    \n",
    "    # Q-Q Plot\n",
    "    stats.probplot(residuals, dist=\"norm\", plot=ax[2])\n",
    "    ax[2].set_title('Q-Q Plot')\n",
    "    \n",
    "    plt.tight_layout()\n",
    "    plt.show()\n",
    "\n",
    "# Function to calculate and print model accuracy metrics\n",
    "def calculate_accuracy(test_data, forecast):\n",
    "    \"\"\"Calculates and prints MSE, MAE, and RMSE.\"\"\"\n",
    "    mse = mean_squared_error(test_data, forecast)\n",
    "    mae = mean_absolute_error(test_data, forecast)\n",
    "    rmse = np.sqrt(mse)\n",
    "    \n",
    "    logging.info(f'MSE: {mse:.3f}')\n",
    "    logging.info(f'MAE: {mae:.3f}')\n",
    "    logging.info(f'RMSE: {rmse:.3f}')\n",
    "    \n",
    "    print(f'MSE: {mse:.3f}')\n",
    "    print(f'MAE: {mae:.3f}')\n",
    "    print(f'RMSE: {rmse:.3f}')\n",
    "\n",
    "# Main function to train and analyze SARIMAX model\n",
    "def main():\n",
    "    # Assuming `stationary_data` is already loaded from previous steps\n",
    "    target_column = 'Close_stationary'\n",
    "    train_start, train_end = '01-04-2019', '31-03-2023'\n",
    "    test_start, test_end = '01-04-2023', '31-03-2024'\n",
    "    \n",
    "    train_data, test_data = split_data(stationary_data[target_column].dropna(), train_start, train_end, test_start, test_end)\n",
    "    \n",
    "    # Train SARIMAX model with assumed p, d, q values and seasonal_order (these should be fine-tuned)\n",
    "    p, d, q, seasonal_order = 1, 1, 1, (1, 1, 1, 12)\n",
    "    model_fit = train_sarimax(train_data, p, d, q, seasonal_order)\n",
    "    \n",
    "    # Forecast for the test period\n",
    "    forecast = model_fit.forecast(steps=len(test_data))\n",
    "    \n",
    "    # Plot residuals\n",
    "    plot_residuals(model_fit.resid)\n",
    "    \n",
    "    # Evaluate accuracy metrics\n",
    "    calculate_accuracy(test_data, forecast)\n",
    "    \n",
    "    # Print SARIMAX summary\n",
    "    print(model_fit.summary())\n",
    "\n",
    "# Run the main function\n",
    "main()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2afff35b-e77a-4fa0-8dfa-3c39a3e6e52b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "anaconda-ai-2024.04-py310",
   "language": "python",
   "name": "conda-env-anaconda-ai-2024.04-py310-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}