CartPole-v0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Mountain Car- v0 solution using action-value neural network function approximator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "from torchvision import transforms\n",
    "from torch.autograd import Variable\n",
    "\n",
    "# One hidden layer function approximator\n",
    "class action_value_function(nn.Module):\n",
    "\n",
    "    def __init__(self,n_observations, n_actions):\n",
    "\n",
    "        super(action_value_function, self).__init__()\n",
    "        self.block = nn.Sequential(\n",
    "            torch.nn.Linear(n_observations, 100, bias = False),\n",
    "            torch.nn.Linear(100, n_actions, bias = False),\n",
    "        )\n",
    "\n",
    "    def forward(self, input):\n",
    "        return self.block(input)\n",
    "    \n",
    "# I tried different weight initializations but found they did not perform well.\n",
    "def weights_init(m):\n",
    "    classname = m.__class__.__name__\n",
    "    if classname.find('Linear') != -1:\n",
    "        nn.init.normal_(m.weight, 0, 1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gym\n",
    "import numpy as np\n",
    "\n",
    "done = False\n",
    "LEARNING_RATE = 0.1\n",
    "DISCOUNT = 1.0\n",
    "EPISODES = 25000\n",
    "SHOW_EVERY = 300\n",
    "BATCH_SIZE = 4\n",
    "rewards_path = 'rewards.npy'\n",
    "\n",
    "\n",
    "env = gym.make(\"CartPole-v0\")\n",
    "env.reset()\n",
    "\n",
    "# Instaciate action_value_function class\n",
    "q_hat = action_value_function(env.observation_space.shape[0], env.action_space.n)\n",
    "q_hat_target = action_value_function(env.observation_space.shape[0], env.action_space.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "observation_space: [-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38] to [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38] | Number of action values: 2\n"
     ]
    }
   ],
   "source": [
    "# Test number of actions and other parameters\n",
    "print('observation_space:',env.observation_space.low,'to',env.observation_space.high,'| Number of action values:',env.action_space.n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-4.8000002e+00, -3.4028235e+38, -4.1887903e-01, -3.4028235e+38],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch.optim as optim\n",
    "\n",
    "criterion = nn.MSELoss()\n",
    "optimizer = optim.SGD(q_hat.parameters(), lr=1e-5)\n",
    "# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=1, gamma=0.9)\n",
    "env.observation_space.low"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-0.01279914 -0.01352579  0.01693123  0.00296156]\n"
     ]
    }
   ],
   "source": [
    "env.close()\n",
    "# cart_position, cart_velocity, angle, pole_tip_velocity = env.reset()\n",
    "print(env.reset())\n",
    "# print('cart_position: %5.2f | cart_velocity: %5.2f | angle: %5.2f | Pole Tip velocity: %5.2f'%(cart_position, cart_velocity, angle, pole_tip_velocity))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Dynamic plots of rewards\n",
    "# import matplotlib.pyplot as plt\n",
    "# import matplotlib.animation as animation\n",
    "# import time\n",
    "\n",
    "\n",
    "# %matplotlib qt\n",
    "\n",
    "# fig = plt.figure()\n",
    "# ax1 = fig.add_subplot(1,1,1)\n",
    "\n",
    "# def animate(i):\n",
    "# #     pullData = open(\"sampleText.txt\",\"r\").read()\n",
    "# #     dataArray = pullData.split('\\n')\n",
    "# #     xar = []\n",
    "# #     yar = []\n",
    "# #     for eachLine in dataArray:\n",
    "# #         if len(eachLine)>1:\n",
    "# #             x,y = eachLine.split(',')\n",
    "# #             xar.append(int(x))\n",
    "# #             yar.append(int(y))\n",
    "#     rewards = np.load(rewards_path)\n",
    "#     ax1.clear()\n",
    "#     ax1.plot(rewards)\n",
    "# ani = animation.FuncAnimation(fig, animate, interval=1000)\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncompatibleKeys(missing_keys=[], unexpected_keys=[])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_loss = 0\n",
    "epsilon = 0.9\n",
    "curr_high_pos = 0\n",
    "curr_low_pos = 0\n",
    "successes = 0\n",
    "UPD = 50\n",
    "upd = 0\n",
    "STEPS = 200\n",
    "ep_rewards = np.array([])\n",
    "q_hat_target.load_state_dict(q_hat.state_dict())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------\n",
      "Episode:     0 | Average Loss: 11.53 | Epsilon: 0.70 | Avg. Reward: 14.06\n",
      "Episode:   100 | Average Loss: 12.04 | Epsilon: 0.70 | Avg. Reward: 14.55\n",
      "Episode:   200 | Average Loss: 12.04 | Epsilon: 0.70 | Avg. Reward: 14.58\n",
      "----------------------------------------------------------------------\n",
      "Episode:   300 | Average Loss: 12.08 | Epsilon: 0.70 | Avg. Reward: 14.49\n",
      "Episode:   400 | Average Loss: 13.00 | Epsilon: 0.70 | Avg. Reward: 15.80\n",
      "Episode:   500 | Average Loss: 12.64 | Epsilon: 0.70 | Avg. Reward: 15.40\n",
      "----------------------------------------------------------------------\n",
      "Episode:   600 | Average Loss: 11.65 | Epsilon: 0.70 | Avg. Reward: 14.21\n",
      "Episode:   700 | Average Loss: 12.59 | Epsilon: 0.70 | Avg. Reward: 14.96\n",
      "Episode:   800 | Average Loss: 13.07 | Epsilon: 0.70 | Avg. Reward: 15.55\n",
      "----------------------------------------------------------------------\n",
      "Episode:   900 | Average Loss: 11.83 | Epsilon: 0.69 | Avg. Reward: 14.50\n",
      "Episode:  1000 | Average Loss: 12.32 | Epsilon: 0.69 | Avg. Reward: 14.93\n",
      "Episode:  1100 | Average Loss: 12.03 | Epsilon: 0.69 | Avg. Reward: 14.76\n",
      "----------------------------------------------------------------------\n",
      "Episode:  1200 | Average Loss: 12.18 | Epsilon: 0.69 | Avg. Reward: 14.93\n",
      "Episode:  1300 | Average Loss: 11.80 | Epsilon: 0.69 | Avg. Reward: 14.27\n",
      "Episode:  1400 | Average Loss: 12.35 | Epsilon: 0.69 | Avg. Reward: 15.28\n",
      "----------------------------------------------------------------------\n",
      "Episode:  1500 | Average Loss: 11.89 | Epsilon: 0.69 | Avg. Reward: 14.36\n",
      "Episode:  1600 | Average Loss: 12.04 | Epsilon: 0.69 | Avg. Reward: 14.16\n",
      "Episode:  1700 | Average Loss: 11.20 | Epsilon: 0.69 | Avg. Reward: 13.54\n",
      "----------------------------------------------------------------------\n",
      "Episode:  1800 | Average Loss: 12.07 | Epsilon: 0.69 | Avg. Reward: 14.36\n",
      "Episode:  1900 | Average Loss: 12.57 | Epsilon: 0.69 | Avg. Reward: 14.90\n",
      "Episode:  2000 | Average Loss: 12.63 | Epsilon: 0.69 | Avg. Reward: 15.17\n",
      "----------------------------------------------------------------------\n",
      "Episode:  2100 | Average Loss: 12.20 | Epsilon: 0.69 | Avg. Reward: 14.44\n",
      "Episode:  2200 | Average Loss: 11.58 | Epsilon: 0.69 | Avg. Reward: 13.61\n",
      "Episode:  2300 | Average Loss: 12.20 | Epsilon: 0.68 | Avg. Reward: 14.74\n",
      "----------------------------------------------------------------------\n",
      "Episode:  2400 | Average Loss: 11.66 | Epsilon: 0.68 | Avg. Reward: 13.84\n",
      "Episode:  2500 | Average Loss: 12.40 | Epsilon: 0.68 | Avg. Reward: 14.36\n",
      "Episode:  2600 | Average Loss: 12.39 | Epsilon: 0.68 | Avg. Reward: 14.60\n",
      "----------------------------------------------------------------------\n",
      "Episode:  2700 | Average Loss: 12.82 | Epsilon: 0.68 | Avg. Reward: 14.67\n",
      "Episode:  2800 | Average Loss: 12.16 | Epsilon: 0.68 | Avg. Reward: 14.18\n",
      "Episode:  2900 | Average Loss: 11.59 | Epsilon: 0.68 | Avg. Reward: 13.69\n",
      "----------------------------------------------------------------------\n",
      "Episode:  3000 | Average Loss: 12.90 | Epsilon: 0.68 | Avg. Reward: 14.64\n",
      "Episode:  3100 | Average Loss: 12.10 | Epsilon: 0.68 | Avg. Reward: 14.44\n",
      "Episode:  3200 | Average Loss: 12.96 | Epsilon: 0.68 | Avg. Reward: 15.72\n",
      "----------------------------------------------------------------------\n",
      "Episode:  3300 | Average Loss: 11.82 | Epsilon: 0.68 | Avg. Reward: 13.93\n",
      "Episode:  3400 | Average Loss: 12.88 | Epsilon: 0.68 | Avg. Reward: 14.79\n",
      "Episode:  3500 | Average Loss: 12.57 | Epsilon: 0.68 | Avg. Reward: 14.80\n",
      "----------------------------------------------------------------------\n",
      "Episode:  3600 | Average Loss: 12.58 | Epsilon: 0.68 | Avg. Reward: 14.67\n",
      "Episode:  3700 | Average Loss: 12.20 | Epsilon: 0.68 | Avg. Reward: 14.71\n",
      "Episode:  3800 | Average Loss: 12.86 | Epsilon: 0.67 | Avg. Reward: 14.86\n",
      "----------------------------------------------------------------------\n",
      "Episode:  3900 | Average Loss: 13.27 | Epsilon: 0.67 | Avg. Reward: 15.59\n",
      "Episode:  4000 | Average Loss: 12.10 | Epsilon: 0.67 | Avg. Reward: 14.03\n",
      "Episode:  4100 | Average Loss: 12.08 | Epsilon: 0.67 | Avg. Reward: 14.32\n",
      "----------------------------------------------------------------------\n",
      "Episode:  4200 | Average Loss: 11.40 | Epsilon: 0.67 | Avg. Reward: 13.35\n",
      "Episode:  4300 | Average Loss: 12.47 | Epsilon: 0.67 | Avg. Reward: 14.36\n",
      "Episode:  4400 | Average Loss: 12.07 | Epsilon: 0.67 | Avg. Reward: 13.94\n",
      "----------------------------------------------------------------------\n",
      "Episode:  4500 | Average Loss: 13.49 | Epsilon: 0.67 | Avg. Reward: 15.61\n",
      "Episode:  4600 | Average Loss: 12.97 | Epsilon: 0.67 | Avg. Reward: 14.89\n",
      "Episode:  4700 | Average Loss: 12.23 | Epsilon: 0.67 | Avg. Reward: 14.12\n",
      "----------------------------------------------------------------------\n",
      "Episode:  4800 | Average Loss: 12.79 | Epsilon: 0.67 | Avg. Reward: 14.40\n",
      "Episode:  4900 | Average Loss: 13.00 | Epsilon: 0.67 | Avg. Reward: 15.14\n",
      "Episode:  5000 | Average Loss: 12.60 | Epsilon: 0.67 | Avg. Reward: 13.93\n",
      "----------------------------------------------------------------------\n",
      "Episode:  5100 | Average Loss: 12.27 | Epsilon: 0.67 | Avg. Reward: 14.09\n",
      "Episode:  5200 | Average Loss: 12.85 | Epsilon: 0.67 | Avg. Reward: 14.89\n",
      "Episode:  5300 | Average Loss: 13.05 | Epsilon: 0.66 | Avg. Reward: 14.59\n",
      "----------------------------------------------------------------------\n",
      "Episode:  5400 | Average Loss: 11.98 | Epsilon: 0.66 | Avg. Reward: 13.79\n",
      "Episode:  5500 | Average Loss: 12.53 | Epsilon: 0.66 | Avg. Reward: 14.07\n",
      "Episode:  5600 | Average Loss: 12.95 | Epsilon: 0.66 | Avg. Reward: 14.57\n",
      "----------------------------------------------------------------------\n",
      "Episode:  5700 | Average Loss: 13.23 | Epsilon: 0.66 | Avg. Reward: 15.14\n",
      "Episode:  5800 | Average Loss: 12.23 | Epsilon: 0.66 | Avg. Reward: 13.95\n",
      "Episode:  5900 | Average Loss: 12.66 | Epsilon: 0.66 | Avg. Reward: 14.71\n",
      "----------------------------------------------------------------------\n",
      "Episode:  6000 | Average Loss: 12.65 | Epsilon: 0.66 | Avg. Reward: 14.38\n",
      "Episode:  6100 | Average Loss: 12.60 | Epsilon: 0.66 | Avg. Reward: 14.39\n",
      "Episode:  6200 | Average Loss: 11.75 | Epsilon: 0.66 | Avg. Reward: 13.63\n",
      "----------------------------------------------------------------------\n",
      "Episode:  6300 | Average Loss: 12.30 | Epsilon: 0.66 | Avg. Reward: 13.76\n",
      "Episode:  6400 | Average Loss: 12.24 | Epsilon: 0.66 | Avg. Reward: 13.96\n",
      "Episode:  6500 | Average Loss: 12.45 | Epsilon: 0.66 | Avg. Reward: 13.94\n",
      "----------------------------------------------------------------------\n",
      "Episode:  6600 | Average Loss: 13.56 | Epsilon: 0.66 | Avg. Reward: 15.05\n",
      "Episode:  6700 | Average Loss: 13.05 | Epsilon: 0.66 | Avg. Reward: 14.74\n",
      "Episode:  6800 | Average Loss: 12.40 | Epsilon: 0.65 | Avg. Reward: 14.07\n",
      "----------------------------------------------------------------------\n",
      "Episode:  6900 | Average Loss: 12.05 | Epsilon: 0.65 | Avg. Reward: 13.42\n",
      "Episode:  7000 | Average Loss: 12.69 | Epsilon: 0.65 | Avg. Reward: 14.27\n",
      "Episode:  7100 | Average Loss: 12.20 | Epsilon: 0.65 | Avg. Reward: 13.86\n",
      "----------------------------------------------------------------------\n",
      "Episode:  7200 | Average Loss: 13.40 | Epsilon: 0.65 | Avg. Reward: 14.85\n",
      "Episode:  7300 | Average Loss: 12.68 | Epsilon: 0.65 | Avg. Reward: 14.31\n",
      "Episode:  7400 | Average Loss: 12.27 | Epsilon: 0.65 | Avg. Reward: 14.07\n",
      "----------------------------------------------------------------------\n",
      "Episode:  7500 | Average Loss: 12.55 | Epsilon: 0.65 | Avg. Reward: 13.92\n",
      "Episode:  7600 | Average Loss: 12.90 | Epsilon: 0.65 | Avg. Reward: 14.19\n",
      "Episode:  7700 | Average Loss: 12.94 | Epsilon: 0.65 | Avg. Reward: 14.12\n",
      "----------------------------------------------------------------------\n",
      "Episode:  7800 | Average Loss: 12.02 | Epsilon: 0.65 | Avg. Reward: 13.49\n",
      "Episode:  7900 | Average Loss: 12.14 | Epsilon: 0.65 | Avg. Reward: 13.41\n",
      "Episode:  8000 | Average Loss: 12.24 | Epsilon: 0.65 | Avg. Reward: 13.50\n",
      "----------------------------------------------------------------------\n",
      "Episode:  8100 | Average Loss: 12.31 | Epsilon: 0.65 | Avg. Reward: 13.88\n",
      "Episode:  8200 | Average Loss: 12.63 | Epsilon: 0.65 | Avg. Reward: 13.92\n",
      "Episode:  8300 | Average Loss: 11.68 | Epsilon: 0.65 | Avg. Reward: 12.97\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------\n",
      "Episode:  8400 | Average Loss: 12.10 | Epsilon: 0.64 | Avg. Reward: 13.42\n",
      "Episode:  8500 | Average Loss: 12.08 | Epsilon: 0.64 | Avg. Reward: 13.44\n",
      "Episode:  8600 | Average Loss: 12.29 | Epsilon: 0.64 | Avg. Reward: 14.03\n",
      "----------------------------------------------------------------------\n",
      "Episode:  8700 | Average Loss: 12.64 | Epsilon: 0.64 | Avg. Reward: 13.79\n",
      "Episode:  8800 | Average Loss: 12.38 | Epsilon: 0.64 | Avg. Reward: 13.41\n",
      "Episode:  8900 | Average Loss: 12.89 | Epsilon: 0.64 | Avg. Reward: 14.03\n",
      "----------------------------------------------------------------------\n",
      "Episode:  9000 | Average Loss: 12.71 | Epsilon: 0.64 | Avg. Reward: 13.98\n",
      "Episode:  9100 | Average Loss: 12.42 | Epsilon: 0.64 | Avg. Reward: 13.42\n",
      "Episode:  9200 | Average Loss: 12.03 | Epsilon: 0.64 | Avg. Reward: 13.23\n",
      "----------------------------------------------------------------------\n",
      "Episode:  9300 | Average Loss: 12.19 | Epsilon: 0.64 | Avg. Reward: 13.57\n",
      "Episode:  9400 | Average Loss: 12.15 | Epsilon: 0.64 | Avg. Reward: 13.44\n",
      "Episode:  9500 | Average Loss: 12.91 | Epsilon: 0.64 | Avg. Reward: 13.80\n",
      "----------------------------------------------------------------------\n",
      "Episode:  9600 | Average Loss: 12.82 | Epsilon: 0.64 | Avg. Reward: 13.99\n",
      "Episode:  9700 | Average Loss: 13.43 | Epsilon: 0.64 | Avg. Reward: 14.70\n",
      "Episode:  9800 | Average Loss: 12.72 | Epsilon: 0.64 | Avg. Reward: 13.83\n",
      "----------------------------------------------------------------------\n",
      "Episode:  9900 | Average Loss: 12.74 | Epsilon: 0.63 | Avg. Reward: 14.22\n",
      "Episode: 10000 | Average Loss: 12.51 | Epsilon: 0.63 | Avg. Reward: 13.69\n",
      "Episode: 10100 | Average Loss: 12.09 | Epsilon: 0.63 | Avg. Reward: 13.63\n",
      "----------------------------------------------------------------------\n",
      "Episode: 10200 | Average Loss: 13.08 | Epsilon: 0.63 | Avg. Reward: 14.48\n",
      "Episode: 10300 | Average Loss: 12.63 | Epsilon: 0.63 | Avg. Reward: 14.03\n",
      "Episode: 10400 | Average Loss: 12.18 | Epsilon: 0.63 | Avg. Reward: 13.29\n",
      "----------------------------------------------------------------------\n",
      "Episode: 10500 | Average Loss: 13.22 | Epsilon: 0.63 | Avg. Reward: 14.59\n",
      "Episode: 10600 | Average Loss: 12.14 | Epsilon: 0.63 | Avg. Reward: 12.91\n",
      "Episode: 10700 | Average Loss: 12.58 | Epsilon: 0.63 | Avg. Reward: 13.77\n",
      "----------------------------------------------------------------------\n",
      "Episode: 10800 | Average Loss: 12.84 | Epsilon: 0.63 | Avg. Reward: 13.75\n",
      "Episode: 10900 | Average Loss: 12.28 | Epsilon: 0.63 | Avg. Reward: 13.87\n",
      "Episode: 11000 | Average Loss: 12.81 | Epsilon: 0.63 | Avg. Reward: 13.88\n",
      "----------------------------------------------------------------------\n",
      "Episode: 11100 | Average Loss: 12.94 | Epsilon: 0.63 | Avg. Reward: 13.45\n",
      "Episode: 11200 | Average Loss: 12.94 | Epsilon: 0.63 | Avg. Reward: 13.64\n",
      "Episode: 11300 | Average Loss: 11.93 | Epsilon: 0.63 | Avg. Reward: 12.77\n",
      "----------------------------------------------------------------------\n",
      "Episode: 11400 | Average Loss: 13.08 | Epsilon: 0.63 | Avg. Reward: 13.85\n",
      "Episode: 11500 | Average Loss: 12.42 | Epsilon: 0.62 | Avg. Reward: 13.56\n",
      "Episode: 11600 | Average Loss: 12.72 | Epsilon: 0.62 | Avg. Reward: 13.35\n",
      "----------------------------------------------------------------------\n",
      "Episode: 11700 | Average Loss: 13.21 | Epsilon: 0.62 | Avg. Reward: 13.64\n",
      "Episode: 11800 | Average Loss: 12.49 | Epsilon: 0.62 | Avg. Reward: 13.49\n",
      "Episode: 11900 | Average Loss: 12.39 | Epsilon: 0.62 | Avg. Reward: 13.35\n",
      "----------------------------------------------------------------------\n",
      "Episode: 12000 | Average Loss: 13.32 | Epsilon: 0.62 | Avg. Reward: 13.88\n",
      "Episode: 12100 | Average Loss: 13.22 | Epsilon: 0.62 | Avg. Reward: 13.65\n",
      "Episode: 12200 | Average Loss: 13.10 | Epsilon: 0.62 | Avg. Reward: 13.73\n",
      "----------------------------------------------------------------------\n",
      "Episode: 12300 | Average Loss: 13.61 | Epsilon: 0.62 | Avg. Reward: 14.00\n",
      "Episode: 12400 | Average Loss: 12.81 | Epsilon: 0.62 | Avg. Reward: 13.19\n",
      "Episode: 12500 | Average Loss: 12.00 | Epsilon: 0.62 | Avg. Reward: 12.76\n",
      "----------------------------------------------------------------------\n",
      "Episode: 12600 | Average Loss: 12.83 | Epsilon: 0.62 | Avg. Reward: 13.57\n",
      "Episode: 12700 | Average Loss: 13.20 | Epsilon: 0.62 | Avg. Reward: 13.47\n",
      "Episode: 12800 | Average Loss: 12.61 | Epsilon: 0.62 | Avg. Reward: 12.91\n",
      "----------------------------------------------------------------------\n",
      "Episode: 12900 | Average Loss: 13.15 | Epsilon: 0.62 | Avg. Reward: 13.22\n",
      "Episode: 13000 | Average Loss: 12.94 | Epsilon: 0.62 | Avg. Reward: 12.94\n",
      "Episode: 13100 | Average Loss: 13.12 | Epsilon: 0.61 | Avg. Reward: 13.50\n",
      "----------------------------------------------------------------------\n",
      "Episode: 13200 | Average Loss: 12.55 | Epsilon: 0.61 | Avg. Reward: 12.86\n",
      "Episode: 13300 | Average Loss: 12.32 | Epsilon: 0.61 | Avg. Reward: 12.82\n",
      "Episode: 13400 | Average Loss: 13.01 | Epsilon: 0.61 | Avg. Reward: 13.65\n",
      "----------------------------------------------------------------------\n",
      "Episode: 13500 | Average Loss: 13.82 | Epsilon: 0.61 | Avg. Reward: 14.04\n",
      "Episode: 13600 | Average Loss: 13.25 | Epsilon: 0.61 | Avg. Reward: 13.90\n",
      "Episode: 13700 | Average Loss: 13.04 | Epsilon: 0.61 | Avg. Reward: 13.78\n",
      "----------------------------------------------------------------------\n",
      "Episode: 13800 | Average Loss: 13.36 | Epsilon: 0.61 | Avg. Reward: 13.33\n",
      "Episode: 13900 | Average Loss: 13.68 | Epsilon: 0.61 | Avg. Reward: 14.18\n",
      "Episode: 14000 | Average Loss: 12.74 | Epsilon: 0.61 | Avg. Reward: 13.37\n",
      "----------------------------------------------------------------------\n",
      "Episode: 14100 | Average Loss: 12.47 | Epsilon: 0.61 | Avg. Reward: 13.03\n",
      "Episode: 14200 | Average Loss: 12.83 | Epsilon: 0.61 | Avg. Reward: 13.26\n",
      "Episode: 14300 | Average Loss: 13.16 | Epsilon: 0.61 | Avg. Reward: 13.14\n",
      "----------------------------------------------------------------------\n",
      "Episode: 14400 | Average Loss: 13.57 | Epsilon: 0.61 | Avg. Reward: 13.76\n",
      "Episode: 14500 | Average Loss: 12.74 | Epsilon: 0.61 | Avg. Reward: 13.29\n",
      "Episode: 14600 | Average Loss: 13.12 | Epsilon: 0.61 | Avg. Reward: 13.19\n",
      "----------------------------------------------------------------------\n",
      "Episode: 14700 | Average Loss: 13.08 | Epsilon: 0.61 | Avg. Reward: 13.47\n",
      "Episode: 14800 | Average Loss: 12.37 | Epsilon: 0.60 | Avg. Reward: 12.78\n",
      "Episode: 14900 | Average Loss: 12.85 | Epsilon: 0.60 | Avg. Reward: 13.06\n",
      "----------------------------------------------------------------------\n",
      "Episode: 15000 | Average Loss: 14.39 | Epsilon: 0.60 | Avg. Reward: 14.13\n",
      "Episode: 15100 | Average Loss: 12.80 | Epsilon: 0.60 | Avg. Reward: 13.00\n",
      "Episode: 15200 | Average Loss: 14.07 | Epsilon: 0.60 | Avg. Reward: 13.52\n",
      "----------------------------------------------------------------------\n",
      "Episode: 15300 | Average Loss: 13.22 | Epsilon: 0.60 | Avg. Reward: 13.06\n",
      "Episode: 15400 | Average Loss: 14.21 | Epsilon: 0.60 | Avg. Reward: 14.26\n",
      "Episode: 15500 | Average Loss: 13.40 | Epsilon: 0.60 | Avg. Reward: 13.26\n",
      "----------------------------------------------------------------------\n",
      "Episode: 15600 | Average Loss: 12.97 | Epsilon: 0.60 | Avg. Reward: 13.27\n",
      "Episode: 15700 | Average Loss: 13.11 | Epsilon: 0.60 | Avg. Reward: 13.46\n",
      "Episode: 15800 | Average Loss: 13.64 | Epsilon: 0.60 | Avg. Reward: 13.54\n",
      "----------------------------------------------------------------------\n",
      "Episode: 15900 | Average Loss: 13.51 | Epsilon: 0.60 | Avg. Reward: 13.39\n",
      "Episode: 16000 | Average Loss: 13.44 | Epsilon: 0.60 | Avg. Reward: 13.15\n",
      "Episode: 16100 | Average Loss: 13.32 | Epsilon: 0.60 | Avg. Reward: 13.18\n",
      "----------------------------------------------------------------------\n",
      "Episode: 16200 | Average Loss: 13.66 | Epsilon: 0.60 | Avg. Reward: 13.91\n",
      "Episode: 16300 | Average Loss: 13.38 | Epsilon: 0.60 | Avg. Reward: 13.43\n",
      "Episode: 16400 | Average Loss: 14.10 | Epsilon: 0.59 | Avg. Reward: 13.99\n",
      "----------------------------------------------------------------------\n",
      "Episode: 16500 | Average Loss: 13.65 | Epsilon: 0.59 | Avg. Reward: 13.48\n",
      "Episode: 16600 | Average Loss: 13.07 | Epsilon: 0.59 | Avg. Reward: 12.90\n",
      "Episode: 16700 | Average Loss: 13.22 | Epsilon: 0.59 | Avg. Reward: 13.33\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------------\n",
      "Episode: 16800 | Average Loss: 13.32 | Epsilon: 0.59 | Avg. Reward: 13.46\n",
      "Episode: 16900 | Average Loss: 13.49 | Epsilon: 0.59 | Avg. Reward: 13.68\n",
      "Episode: 17000 | Average Loss: 13.11 | Epsilon: 0.59 | Avg. Reward: 13.12\n",
      "----------------------------------------------------------------------\n",
      "Episode: 17100 | Average Loss: 13.01 | Epsilon: 0.59 | Avg. Reward: 13.03\n",
      "Episode: 17200 | Average Loss: 13.82 | Epsilon: 0.59 | Avg. Reward: 13.71\n",
      "Episode: 17300 | Average Loss: 14.06 | Epsilon: 0.59 | Avg. Reward: 13.70\n",
      "----------------------------------------------------------------------\n",
      "Episode: 17400 | Average Loss: 12.99 | Epsilon: 0.59 | Avg. Reward: 13.07\n",
      "Episode: 17500 | Average Loss: 13.56 | Epsilon: 0.59 | Avg. Reward: 13.57\n",
      "Episode: 17600 | Average Loss: 13.49 | Epsilon: 0.59 | Avg. Reward: 13.53\n",
      "----------------------------------------------------------------------\n",
      "Episode: 17700 | Average Loss: 12.66 | Epsilon: 0.59 | Avg. Reward: 12.73\n",
      "Episode: 17800 | Average Loss: 13.38 | Epsilon: 0.59 | Avg. Reward: 12.94\n",
      "Episode: 17900 | Average Loss: 13.77 | Epsilon: 0.59 | Avg. Reward: 13.40\n",
      "----------------------------------------------------------------------\n",
      "Episode: 18000 | Average Loss: 13.72 | Epsilon: 0.59 | Avg. Reward: 13.08\n",
      "Episode: 18100 | Average Loss: 14.54 | Epsilon: 0.58 | Avg. Reward: 14.05\n",
      "Episode: 18200 | Average Loss: 14.09 | Epsilon: 0.58 | Avg. Reward: 13.33\n",
      "----------------------------------------------------------------------\n",
      "Episode: 18300 | Average Loss: 13.65 | Epsilon: 0.58 | Avg. Reward: 12.88\n",
      "Episode: 18400 | Average Loss: 13.09 | Epsilon: 0.58 | Avg. Reward: 12.67\n",
      "Episode: 18500 | Average Loss: 14.12 | Epsilon: 0.58 | Avg. Reward: 13.62\n",
      "----------------------------------------------------------------------\n",
      "Episode: 18600 | Average Loss: 12.76 | Epsilon: 0.58 | Avg. Reward: 12.26\n",
      "Episode: 18700 | Average Loss: 13.19 | Epsilon: 0.58 | Avg. Reward: 12.67\n",
      "Episode: 18800 | Average Loss: 13.02 | Epsilon: 0.58 | Avg. Reward: 12.51\n",
      "----------------------------------------------------------------------\n",
      "Episode: 18900 | Average Loss: 13.84 | Epsilon: 0.58 | Avg. Reward: 13.08\n",
      "Episode: 19000 | Average Loss: 15.12 | Epsilon: 0.58 | Avg. Reward: 14.37\n",
      "Episode: 19100 | Average Loss: 13.92 | Epsilon: 0.58 | Avg. Reward: 13.35\n",
      "----------------------------------------------------------------------\n",
      "Episode: 19200 | Average Loss: 14.08 | Epsilon: 0.58 | Avg. Reward: 13.42\n",
      "Episode: 19300 | Average Loss: 13.57 | Epsilon: 0.58 | Avg. Reward: 13.42\n",
      "Episode: 19400 | Average Loss: 12.95 | Epsilon: 0.58 | Avg. Reward: 12.78\n",
      "----------------------------------------------------------------------\n",
      "Episode: 19500 | Average Loss: 13.63 | Epsilon: 0.58 | Avg. Reward: 12.89\n",
      "Episode: 19600 | Average Loss: 13.81 | Epsilon: 0.58 | Avg. Reward: 13.50\n",
      "Episode: 19700 | Average Loss: 13.44 | Epsilon: 0.58 | Avg. Reward: 12.76\n",
      "----------------------------------------------------------------------\n",
      "Episode: 19800 | Average Loss: 13.40 | Epsilon: 0.58 | Avg. Reward: 12.78\n",
      "Episode: 19900 | Average Loss: 13.47 | Epsilon: 0.57 | Avg. Reward: 12.52\n",
      "Episode: 20000 | Average Loss: 14.57 | Epsilon: 0.57 | Avg. Reward: 13.70\n",
      "----------------------------------------------------------------------\n",
      "Episode: 20100 | Average Loss: 14.18 | Epsilon: 0.57 | Avg. Reward: 13.49\n",
      "Episode: 20200 | Average Loss: 12.85 | Epsilon: 0.57 | Avg. Reward: 12.24\n",
      "Episode: 20300 | Average Loss: 13.32 | Epsilon: 0.57 | Avg. Reward: 13.24\n",
      "----------------------------------------------------------------------\n",
      "Episode: 20400 | Average Loss: 13.45 | Epsilon: 0.57 | Avg. Reward: 13.13\n",
      "Episode: 20500 | Average Loss: 14.23 | Epsilon: 0.57 | Avg. Reward: 13.61\n",
      "Episode: 20600 | Average Loss: 12.43 | Epsilon: 0.57 | Avg. Reward: 12.22\n",
      "----------------------------------------------------------------------\n",
      "Episode: 20700 | Average Loss: 13.27 | Epsilon: 0.57 | Avg. Reward: 12.59\n",
      "Episode: 20800 | Average Loss: 13.58 | Epsilon: 0.57 | Avg. Reward: 12.93\n",
      "Episode: 20900 | Average Loss: 14.42 | Epsilon: 0.57 | Avg. Reward: 13.38\n",
      "----------------------------------------------------------------------\n",
      "Episode: 21000 | Average Loss: 13.07 | Epsilon: 0.57 | Avg. Reward: 12.53\n",
      "Episode: 21100 | Average Loss: 14.33 | Epsilon: 0.57 | Avg. Reward: 13.79\n",
      "Episode: 21200 | Average Loss: 13.70 | Epsilon: 0.57 | Avg. Reward: 13.25\n",
      "----------------------------------------------------------------------\n",
      "Episode: 21300 | Average Loss: 13.19 | Epsilon: 0.57 | Avg. Reward: 13.01\n",
      "Episode: 21400 | Average Loss: 13.23 | Epsilon: 0.57 | Avg. Reward: 12.53\n",
      "Episode: 21500 | Average Loss: 12.56 | Epsilon: 0.57 | Avg. Reward: 12.41\n",
      "----------------------------------------------------------------------\n",
      "Episode: 21600 | Average Loss: 14.15 | Epsilon: 0.56 | Avg. Reward: 13.77\n",
      "Episode: 21700 | Average Loss: 13.96 | Epsilon: 0.56 | Avg. Reward: 12.99\n",
      "Episode: 21800 | Average Loss: 14.08 | Epsilon: 0.56 | Avg. Reward: 13.23\n",
      "----------------------------------------------------------------------\n",
      "Episode: 21900 | Average Loss: 13.60 | Epsilon: 0.56 | Avg. Reward: 12.59\n",
      "Episode: 22000 | Average Loss: 13.59 | Epsilon: 0.56 | Avg. Reward: 12.94\n",
      "Episode: 22100 | Average Loss: 13.57 | Epsilon: 0.56 | Avg. Reward: 12.95\n",
      "----------------------------------------------------------------------\n",
      "Episode: 22200 | Average Loss: 13.50 | Epsilon: 0.56 | Avg. Reward: 12.83\n",
      "Episode: 22300 | Average Loss: 14.36 | Epsilon: 0.56 | Avg. Reward: 13.15\n",
      "Episode: 22400 | Average Loss: 13.41 | Epsilon: 0.56 | Avg. Reward: 12.29\n",
      "----------------------------------------------------------------------\n",
      "Episode: 22500 | Average Loss: 13.70 | Epsilon: 0.56 | Avg. Reward: 12.55\n",
      "Episode: 22600 | Average Loss: 13.53 | Epsilon: 0.56 | Avg. Reward: 13.04\n",
      "Episode: 22700 | Average Loss: 13.49 | Epsilon: 0.56 | Avg. Reward: 12.81\n",
      "----------------------------------------------------------------------\n",
      "Episode: 22800 | Average Loss: 13.12 | Epsilon: 0.56 | Avg. Reward: 12.18\n",
      "Episode: 22900 | Average Loss: 14.54 | Epsilon: 0.56 | Avg. Reward: 13.24\n",
      "Episode: 23000 | Average Loss: 13.24 | Epsilon: 0.56 | Avg. Reward: 12.29\n",
      "----------------------------------------------------------------------\n",
      "Episode: 23100 | Average Loss: 14.33 | Epsilon: 0.56 | Avg. Reward: 13.07\n",
      "Episode: 23200 | Average Loss: 12.70 | Epsilon: 0.56 | Avg. Reward: 12.14\n",
      "Episode: 23300 | Average Loss: 13.62 | Epsilon: 0.56 | Avg. Reward: 12.82\n",
      "----------------------------------------------------------------------\n",
      "Episode: 23400 | Average Loss: 14.73 | Epsilon: 0.55 | Avg. Reward: 13.06\n",
      "Episode: 23500 | Average Loss: 15.18 | Epsilon: 0.55 | Avg. Reward: 13.89\n",
      "Episode: 23600 | Average Loss: 14.28 | Epsilon: 0.55 | Avg. Reward: 12.80\n",
      "----------------------------------------------------------------------\n",
      "Episode: 23700 | Average Loss: 14.56 | Epsilon: 0.55 | Avg. Reward: 13.25\n",
      "Episode: 23800 | Average Loss: 14.19 | Epsilon: 0.55 | Avg. Reward: 12.71\n",
      "Episode: 23900 | Average Loss: 14.67 | Epsilon: 0.55 | Avg. Reward: 13.32\n",
      "----------------------------------------------------------------------\n",
      "Episode: 24000 | Average Loss: 14.12 | Epsilon: 0.55 | Avg. Reward: 13.11\n",
      "Episode: 24100 | Average Loss: 14.19 | Epsilon: 0.55 | Avg. Reward: 12.97\n",
      "Episode: 24200 | Average Loss: 13.16 | Epsilon: 0.55 | Avg. Reward: 12.17\n",
      "----------------------------------------------------------------------\n",
      "Episode: 24300 | Average Loss: 13.97 | Epsilon: 0.55 | Avg. Reward: 12.75\n",
      "Episode: 24400 | Average Loss: 13.90 | Epsilon: 0.55 | Avg. Reward: 12.48\n",
      "Episode: 24500 | Average Loss: 14.60 | Epsilon: 0.55 | Avg. Reward: 13.41\n",
      "----------------------------------------------------------------------\n",
      "Episode: 24600 | Average Loss: 14.11 | Epsilon: 0.55 | Avg. Reward: 12.87\n",
      "Episode: 24700 | Average Loss: 13.64 | Epsilon: 0.55 | Avg. Reward: 12.74\n",
      "Episode: 24800 | Average Loss: 14.50 | Epsilon: 0.55 | Avg. Reward: 13.44\n",
      "----------------------------------------------------------------------\n",
      "Episode: 24900 | Average Loss: 13.93 | Epsilon: 0.55 | Avg. Reward: 12.91\n"
     ]
    }
   ],
   "source": [
    "# Train agent\n",
    "import random\n",
    "\n",
    "for episode in range(EPISODES):\n",
    "   \n",
    "    state = torch.from_numpy(env.reset()).float()\n",
    "    done = False\n",
    "    ep_reward = 0\n",
    "    \n",
    "    if episode%SHOW_EVERY == 0:\n",
    "        print('-'*70)\n",
    "        RENDER = True\n",
    "    else:\n",
    "        RENDER = False\n",
    "\n",
    "    running_loss = 0\n",
    "    cont = 0\n",
    "\n",
    "#     for step in range(STEPS):\n",
    "    while not done:\n",
    "        if RENDER:\n",
    "            env.render()\n",
    "        upd += 1;\n",
    "        if upd%UPD == 0:\n",
    "            q_hat_target.load_state_dict(q_hat.state_dict())\n",
    "            upd = 0\n",
    "            \n",
    "        q = q_hat(Variable(state))\n",
    "        \n",
    "        if random.random() < epsilon:\n",
    "            action = env.action_space.sample()\n",
    "        else:\n",
    "            action = torch.argmax(q).item()\n",
    "        \n",
    "        next_state, reward, done, _ = env.step(action)\n",
    "        next_state = torch.from_numpy(next_state).float()\n",
    "        \n",
    "                    \n",
    "#         print(count)\n",
    "             \n",
    "#         if next_state[0] >= 2.4 or next_state[0] <= -2.4:\n",
    "#             reward -= .1\n",
    "#         if next_state[2] >= 12 or next_state[2] < -12:\n",
    "#             reward -= 1\n",
    "            \n",
    "        q_target = q.clone()\n",
    "#         q_target = Variable(q_target.data)\n",
    "        \n",
    "        q_target[action] = torch.tensor(reward) + DISCOUNT * torch.max(q_hat_target(next_state))\n",
    "        if done:\n",
    "            q_target[action] = torch.tensor(reward)\n",
    "            \n",
    "            \n",
    "        # Calculate loss\n",
    "        loss = criterion(q, q_target.detach())\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        state = next_state\n",
    "        \n",
    "        ep_reward += reward\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "    epsilon *= .99999\n",
    "#     scheduler.step()\n",
    "    ep_rewards = np.append(ep_rewards, ep_reward)\n",
    "\n",
    "    avg_loss += running_loss\n",
    "    env.close()\n",
    "    if episode%100 == 0:\n",
    "        print('Episode: %5d | Average Loss: %5.2f | Epsilon: %4.2f | Avg. Reward: %5.2f'%(episode, avg_loss/100, epsilon,np.mean(ep_rewards[-100:])))\n",
    "        avg_loss = 0\n",
    "        ep_reward = 00\n",
    "        np.save(rewards_path, ep_rewards)\n",
    "\n",
    "# env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Save model\n",
    "# torch.save(q_hat.state_dict(),'./models/cartpole-v0.pth')\n",
    "# # q = action_value_function()\n",
    "# q.load_state_dict(torch.load('./models/trained/cartpole-v0.pth'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test your policy\n",
    "env.close()\n",
    "state = torch.from_numpy(env.reset()).float()\n",
    "done = False \n",
    "#             pri\n",
    "EPS_TEST = 0.05\n",
    "tot_reward = 0\n",
    "for step in range(200):\n",
    "    env.render()\n",
    "    q = q_hat((state))\n",
    "    action = torch.argmax(q).item()\n",
    "#     if random.random() < EPS_TEST:\n",
    "#         action = env.action_space.sample() \n",
    "    next_state, reward, done, _ = env.step(action)\n",
    "    tot_reward += reward\n",
    "    next_state = torch.from_numpy(next_state).float()\n",
    "    state = next_state\n",
    "#     if reward == 1:\n",
    "#         print('Hell Yes!')\n",
    "        \n",
    "print(tot_reward)\n",
    "env.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_hat_target.load_state_dict('./models/cartpole-v0.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "q_target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}