first.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pandas import read_csv\n",
    "from os import path\n",
    "from os import listdir, chdir\n",
    "from collections import OrderedDict\n",
    "from pandas import DataFrame\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5611086346\n"
     ]
    }
   ],
   "source": [
    "task_events_csv_colnames = ['time', 'missing', 'job_id', 'task_idx', 'machine_id', 'event_type', 'user', 'sched_cls', \n",
    "                            'priority', 'cpu_requested', 'mem_requested', 'disk', 'restriction']         \n",
    "task_event_df = read_csv(path.join('task_events', 'part-00000-of-00500.csv.gz'), header=None, index_col=False, \n",
    "                         compression='gzip', names=task_events_csv_colnames)\n",
    "\n",
    "maxtime_stamp = max(task_event_df['time'])\n",
    "print maxtime_stamp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "287272106\n"
     ]
    }
   ],
   "source": [
    "from random import randint, sample, seed\n",
    "seed(83)\n",
    "# sample_moments = range(0,maxtime_stamp,1000000) # lay mau theo giay\n",
    "sample_moments = sorted(sample(xrange(maxtime_stamp), 200))\n",
    "snapshot_moment = randint(0, maxtime_stamp)  # lay mau ngau nhien 1 thoi diem\n",
    "print snapshot_moment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-5-dbf8968e8f1d>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     14\u001b[0m                               names=task_events_csv_colnames)\n\u001b[0;32m     15\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 16\u001b[1;33m     \u001b[1;32mfor\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mevent\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mtask_events_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0miterrows\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     17\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     18\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcurrent_sample_moment\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mevent\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mcurrent_sample_moment\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36miterrows\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    529\u001b[0m         \u001b[0mcolumns\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    530\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 531\u001b[1;33m             \u001b[0ms\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSeries\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mv\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mk\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    532\u001b[0m             \u001b[1;32myield\u001b[0m \u001b[0mk\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0ms\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    533\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/series.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, index, dtype, name, copy, fastpath)\u001b[0m\n\u001b[0;32m    222\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mSingleBlockManager\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfastpath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    223\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 224\u001b[1;33m         \u001b[0mgeneric\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mNDFrame\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfastpath\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mTrue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    225\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    226\u001b[0m         \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'name'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, axes, copy, dtype, fastpath)\u001b[0m\n\u001b[0;32m    100\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    101\u001b[0m         \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'is_copy'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 102\u001b[1;33m         \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'_data'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    103\u001b[0m         \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'_item_cache'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m{\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    104\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# doc du lieu tu task_event, tinh tong tat ca cac CPU va RAM request cho tat cac cac job tai tung thoi diem\n",
    "#\n",
    "tasks_dict = {}\n",
    "samples_dicts = OrderedDict([])\n",
    "sample_moments_iterator = iter(sample_moments)\n",
    "current_sample_moment = next(sample_moments_iterator)\n",
    "tasks_df = None # day la fram ung voi lai 1 thoi diem lay mau ngau nhien (snapshot_moment)\n",
    "# Not the most elegant code I've ever written...\n",
    "# %%time\n",
    "for fn in sorted(listdir('task_events')):\n",
    "    \n",
    "    fp = path.join('task_events', fn)\n",
    "    task_events_df = read_csv(fp, header=None, index_col=False, compression='gzip', \n",
    "                              names=task_events_csv_colnames)\n",
    "    \n",
    "    for index, event in task_events_df.iterrows():\n",
    "        \n",
    "        if current_sample_moment is not None and event['time'] > current_sample_moment:\n",
    "            tmp_tasks_df = DataFrame(tasks_dict.values())\n",
    "            samples_dicts[current_sample_moment] = ({'time' : current_sample_moment, \n",
    "                                                     'cpu_requested' : (tmp_tasks_df['cpu_requested']).sum(), \n",
    "                                                     'mem_requested' : (tmp_tasks_df['mem_requested']).sum()})\n",
    "            try:\n",
    "                current_sample_moment = next(sample_moments_iterator)\n",
    "            except StopIteration:\n",
    "                current_sample_moment = None\n",
    "                \n",
    "        if tasks_df is None and event['time'] > snapshot_moment:\n",
    "            tasks_df = DataFrame(tasks_dict.values())\n",
    "            \n",
    "        if event['event_type'] in [0, 7, 8]:\n",
    "            tasks_dict[(event['job_id'], event['task_idx'])] = {'task_id' : (event['job_id'], event['task_idx']),\n",
    "                                                                'machine_id' : event['machine_id'],\n",
    "                                                                'cpu_requested' : event['cpu_requested'], \n",
    "                                                                'mem_requested' : event['mem_requested']}\n",
    "        elif event['event_type'] in [2, 3, 4, 5, 6]:\n",
    "            del tasks_dict[(event['job_id'], event['task_idx'])]\n",
    "            \n",
    "    if tasks_df is not None and current_sample_moment is None:\n",
    "        break\n",
    "                    \n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "\n",
    "print tasks_df.info()\n",
    "print samples_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], samples_df['cpu_requested'], label='cpu requested')\n",
    "ax.plot(samples_df['time'], samples_df['mem_requested'], label='mem requested')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "machines_dict = {}\n",
    "sample_moments_iterator = iter(sample_moments)\n",
    "current_sample_moment = next(sample_moments_iterator)\n",
    "machines_df = None\n",
    "\n",
    "machine_events_csv_colnames=['time', 'machine_id', 'event_type', 'platform_id', 'cpu', 'mem']\n",
    "\n",
    "# %%time\n",
    "for fn in sorted(listdir('machine_events')):\n",
    "    fp = path.join('machine_events', fn)\n",
    "    machine_events_df = read_csv(fp, header=None, index_col=False, compression='gzip', names=machine_events_csv_colnames)\n",
    "    for index, event in machine_events_df.iterrows():\n",
    "        \n",
    "        if current_sample_moment is not None and event['time'] > current_sample_moment:\n",
    "            tmp_machines_df = DataFrame(machines_dict.values())\n",
    "            samples_dicts[current_sample_moment].update({'cpu_available' : (tmp_machines_df['cpu']).sum(), \n",
    "                                                         'mem_available' : (tmp_machines_df['mem']).sum()})\n",
    "            try:\n",
    "                current_sample_moment = next(sample_moments_iterator)\n",
    "            except StopIteration:\n",
    "                current_sample_moment = None\n",
    "                \n",
    "        if machines_df is None and event['time'] > snapshot_moment: \n",
    "            machines_df = DataFrame(machines_dict.values())\n",
    "            \n",
    "        if event['event_type'] in [0, 2]:\n",
    "            machines_dict[event['machine_id']] = {'machine_id' : event['machine_id'], \n",
    "                                                  'cpu' : event['cpu'], 'mem' : event['mem']}\n",
    "        elif event['event_type'] in [1]:\n",
    "            del machines_dict[event['machine_id']]\n",
    "\n",
    "    if machines_df is not None and current_sample_moment is None:\n",
    "        break\n",
    "                    \n",
    "machines_df = DataFrame(machines_dict.values())\n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "\n",
    "print machines_df.info()\n",
    "print\n",
    "print samples_df.info()\n",
    "print"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "u'no item named cpu_available'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-9-4206c744d679>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      2\u001b[0m \u001b[0max\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfig\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0madd_subplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;36m111\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'cpu_requested'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'cpu requested'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 4\u001b[1;33m \u001b[0max\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'cpu_available'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'cpu available'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      5\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mem_requested'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'mem requested'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      6\u001b[0m \u001b[0max\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mplot\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0msamples_df\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'mem_available'\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m'mem available'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   1656\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1657\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1658\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1659\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1660\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_getitem_column\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_getitem_column\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m   1663\u001b[0m         \u001b[1;31m# get column\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1664\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1665\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_get_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1666\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1667\u001b[0m         \u001b[1;31m# duplicate columns & possible reduce dimensionaility\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/generic.pyc\u001b[0m in \u001b[0;36m_get_item_cache\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m   1003\u001b[0m         \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcache\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1004\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mres\u001b[0m \u001b[1;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1005\u001b[1;33m             \u001b[0mvalues\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1006\u001b[0m             \u001b[0mres\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_box_item_values\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalues\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1007\u001b[0m             \u001b[0mcache\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mres\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mget\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m   2872\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget_for_nan_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2873\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 2874\u001b[1;33m             \u001b[0m_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mblock\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_find_block\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   2875\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mblock\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   2876\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36m_find_block\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m   3184\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3185\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_find_block\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3186\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_check_have\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3187\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mblock\u001b[0m \u001b[1;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3188\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mitem\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mblock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36m_check_have\u001b[1;34m(self, item)\u001b[0m\n\u001b[0;32m   3191\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_check_have\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3192\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mitem\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3193\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'no item named %s'\u001b[0m \u001b[1;33m%\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpprint_thing\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mitem\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3194\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3195\u001b[0m     def reindex_axis(self, new_axis, indexer=None, method=None, axis=0,\n",
      "\u001b[1;31mKeyError\u001b[0m: u'no item named cpu_available'"
     ]
    }
   ],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], samples_df['cpu_requested'], label='cpu requested')\n",
    "ax.plot(samples_df['time'], samples_df['cpu_available'], label='cpu available')\n",
    "ax.plot(samples_df['time'], samples_df['mem_requested'], label='mem requested')\n",
    "ax.plot(samples_df['time'], samples_df['mem_available'], label='mem available')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nInt64Index: 200 entries, 0 to 199\nData columns (total 7 columns):\ncpu_available    200 non-null float64\ncpu_requested    200 non-null float64\ncpu_usage        200 non-null float64\nmem_available    200 non-null float64\nmem_requested    200 non-null float64\nmem_usage        200 non-null float64\ntime             200 non-null int64\ndtypes: float64(6), int64(1)None\n<class 'pandas.core.frame.DataFrame'>\nInt64Index: 142830 entries, 0 to 142829\nData columns (total 8 columns):\ncpu_requested    142543 non-null float64\ncpu_usage        142830 non-null float64\nin_events        142830 non-null bool\nin_usage         142830 non-null bool\nmachine_id       86485 non-null float64\nmem_requested    142543 non-null float64\nmem_usage        142830 non-null float64\ntask_id          142830 non-null object\ndtypes: bool(2), float64(5), object(1)"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "task_usage_csv_colnames=['starttime', 'endtime', 'job_id', 'task_idx', 'machine_id', 'cpu_usage', 'mem_usage', \n",
    "                         'assigned_mem', 'unmapped_cache_usage', 'page_cache_usage', 'max_mem_usage', 'disk_io_time', \n",
    "                         'max_disk_space', 'max_cpu_usage', 'max_disk_io_time', 'cpi', 'mai', 'sampling_rate', 'agg_type']\n",
    "\n",
    "for moment in samples_dicts:\n",
    "    samples_dicts[moment].update({'cpu_usage' : 0.0, 'mem_usage' : 0.0})\n",
    "    \n",
    "for task in tasks_dict:\n",
    "    tasks_dict[task].update({'cpu_usage' : 0.0, 'mem_usage' : 0.0, 'in_events' : True, 'in_usage' : False})\n",
    "    \n",
    "# %%time\n",
    "for fn in sorted(listdir('task_usage')):\n",
    "\n",
    "    fp = path.join('task_usage', fn)\n",
    "    task_usage_df = read_csv(fp, header=None, index_col=False, compression='gzip', names=task_usage_csv_colnames)\n",
    "\n",
    "    laststart = max(task_usage_df['starttime'])\n",
    "    firststart = min(task_usage_df['starttime'])\n",
    "    # if laststart > max(sample_moments) and laststart > snapshot_moment:\n",
    "    #     break\n",
    "    if firststart > max(sample_moments) and firststart > snapshot_moment:\n",
    "        break\n",
    "    for moment in samples_dicts:\n",
    "        task_usage_moment_df = task_usage_df[(task_usage_df['starttime'] <= moment) & \n",
    "                                             (moment <= task_usage_df['endtime'])]\n",
    "        # print task_usage_moment_df.info()\n",
    "        samples_dicts[moment]['cpu_usage'] += sum(task_usage_moment_df['cpu_usage'])\n",
    "        samples_dicts[moment]['mem_usage'] += sum(task_usage_moment_df['mem_usage'])\n",
    "\n",
    "    task_usage_snapshot_df = task_usage_df[(task_usage_df['starttime'] <= snapshot_moment) &\n",
    "                                           (snapshot_moment <= task_usage_df['endtime'])]\n",
    "    for index, usage in task_usage_snapshot_df.iterrows():\n",
    "        task_id = (usage['job_id'], usage['task_idx'])\n",
    "        if task_id in tasks_dict:\n",
    "            tasks_dict[task_id].update({'cpu_usage' : usage['cpu_usage'], 'mem_usage' : usage['mem_usage'], \n",
    "                                        'in_events': True, 'in_usage' : True})\n",
    "        else:\n",
    "            tasks_dict[task_id] = {'cpu_requested' : 0.0, 'mem_requested' : 0.0, \n",
    "                                   'cpu_usage' : usage['cpu_usage'], 'mem_usage' : usage['mem_usage'],\n",
    "                                   'in_events' : False, 'in_usage' : True}\n",
    "\n",
    "\n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "print samples_df.info()\n",
    "tasks_df = DataFrame(tasks_dict.values())\n",
    "print tasks_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], samples_df['cpu_requested'], label='cpu requested')\n",
    "ax.plot(samples_df['time'], samples_df['cpu_available'], label='cpu available')\n",
    "ax.plot(samples_df['time'], samples_df['cpu_usage'], label='cpu usage')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], samples_df['mem_requested'], label='mem requested')\n",
    "ax.plot(samples_df['time'], samples_df['mem_available'], label='mem available')\n",
    "ax.plot(samples_df['time'], samples_df['mem_usage'], label='mem usage')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "demand_pct = [100.0 * cpur / cpua for cpur, cpua in zip(samples_df['cpu_requested'], samples_df['cpu_available'])]\n",
    "usage_pct = [100.0 * cpuu / cpua for cpuu, cpua in zip(samples_df['cpu_usage'], samples_df['cpu_available'])]\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], demand_pct, label='cpu % demand')\n",
    "ax.plot(samples_df['time'], usage_pct, label='cpu % usage')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "demand_pct = [100.0 * memr / mema for memr, mema in zip(samples_df['mem_requested'], samples_df['mem_available'])]\n",
    "usage_pct = [100.0 * memu / mema for memu, mema in zip(samples_df['mem_usage'], samples_df['mem_available'])]\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], demand_pct, label='mem % demand')\n",
    "ax.plot(samples_df['time'], usage_pct, label='mem % usage')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nInt64Index: 200 entries, 0 to 199\nData columns (total 5 columns):\ncpu_requested               200 non-null float64\ndisk                        200 non-null float64\nmem_requested               200 non-null float64\nnumber_of_task_requested    200 non-null int64\ntime                        200 non-null int64\ndtypes: float64(3), int64(2)None\n"
     ]
    }
   ],
   "source": [
    "# doc du lieu tu task_event, chon mot Machine ID\n",
    "# roi tinh tong tat ca cac CPU va RAM request cho tat cac cac job tai tung thoi diem\n",
    "machine_id = 1268205\n",
    "tasks_dict = {}\n",
    "samples_dicts = OrderedDict([])\n",
    "sample_moments_iterator = iter(sample_moments)\n",
    "current_sample_moment = next(sample_moments_iterator)\n",
    "tasks_df = None # day la fram ung voi lai 1 thoi diem lay mau ngau nhien (snapshot_moment)\n",
    "# Not the most elegant code I've ever written...\n",
    "# %%time\n",
    "task_events_csv_colnames = ['time', 'missing', 'job_id', 'task_idx', 'machine_id', 'event_type', 'user', 'sched_cls', \n",
    "                            'priority', 'cpu_requested', 'mem_requested', 'disk', 'restriction']  \n",
    "for fn in sorted(listdir('task_events')):\n",
    "    \n",
    "    fp = path.join('task_events', fn)\n",
    "    print 'reading file ' + fp\n",
    "    task_events_df = read_csv(fp, header=None, index_col=False, compression='gzip', \n",
    "                              names=task_events_csv_colnames)\n",
    "    \n",
    "    for index, event in task_events_df.iterrows():\n",
    "        \n",
    "        if current_sample_moment is not None and event['time'] > current_sample_moment:\n",
    "            tmp_tasks_df = DataFrame(tasks_dict.values())\n",
    "            tmp_tasks_df = tmp_tasks_df[tmp_tasks_df['machine_id']==machine_id]\n",
    "            # dong duoi nay co the toi uu hon\n",
    "            samples_dicts[current_sample_moment] = ({'time' : current_sample_moment, \n",
    "                                                     'cpu_requested' : (tmp_tasks_df['cpu_requested']).sum(), \n",
    "                                                     'mem_requested' : (tmp_tasks_df['mem_requested']).sum(),\n",
    "                                                     'disk_space_requested': (tmp_tasks_df['disk']).sum(),\n",
    "                                                     'number_of_task_requested': len(tmp_tasks_df['machine_id'])})\n",
    "            try:\n",
    "                current_sample_moment = next(sample_moments_iterator)\n",
    "            except StopIteration:\n",
    "                current_sample_moment = None\n",
    "                \n",
    "\n",
    "        # if event['machine_id']== machine_id:\n",
    "        if event['event_type'] in [0, 7, 8]:\n",
    "            tasks_dict[(event['job_id'], event['task_idx'])] = {'task_id' : (event['job_id'], event['task_idx']),\n",
    "                                                                    'machine_id' : event['machine_id'],\n",
    "                                                                    'cpu_requested' : event['cpu_requested'], \n",
    "                                                                    'mem_requested' : event['mem_requested'],\n",
    "                                                                    'disk': event['disk']}\n",
    "        elif event['event_type'] in [2, 3, 4, 5, 6]:\n",
    "            try:\n",
    "                del tasks_dict[(event['job_id'], event['task_idx'])]\n",
    "            except:\n",
    "                    # print \"loi:\\n\"\n",
    "                    # print event\n",
    "                pass\n",
    "                    \n",
    "    if tasks_df is not None and current_sample_moment is None:\n",
    "        break\n",
    "                    \n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "print samples_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# %%time\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], samples_df['cpu_requested'], label='cpu requested')\n",
    "ax.plot(samples_df['time'], samples_df['mem_requested'], label='mem requested')\n",
    "ax.plot(samples_df['time'], samples_df['disk_space_requested'], label='disk space requested')\n",
    "# ax.plot(samples_df['time'], samples_df['number_of_task_requested'], label='number of task')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.title('machine ID = ' +str(machine_id))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nInt64Index: 200 entries, 0 to 199\nData columns (total 5 columns):\ncpu_available    200 non-null float64\ncpu_requested    200 non-null float64\nmem_available    200 non-null float64\nmem_requested    200 non-null float64\ntime             200 non-null int64\ndtypes: float64(4), int64(1)None\n\n"
     ]
    }
   ],
   "source": [
    "#lay thong tin CPU, RAM available cua machine\n",
    "machine_id = 1268205\n",
    "machines_dict = {}\n",
    "sample_moments_iterator = iter(sample_moments)\n",
    "current_sample_moment = next(sample_moments_iterator)\n",
    "machines_df = None\n",
    "\n",
    "machine_events_csv_colnames=['time', 'machine_id', 'event_type', 'platform_id', 'cpu', 'mem']\n",
    "\n",
    "# %%time\n",
    "for fn in sorted(listdir('machine_events')):\n",
    "    fp = path.join('machine_events', fn)\n",
    "    machine_events_df = read_csv(fp, header=None, index_col=False, compression='gzip', names=machine_events_csv_colnames)\n",
    "    for index, event in machine_events_df.iterrows():\n",
    "        \n",
    "        if current_sample_moment is not None and event['time'] > current_sample_moment:\n",
    "            tmp_machines_df = DataFrame(machines_dict.values())\n",
    "            if not tmp_machines_df.empty:\n",
    "                samples_dicts[current_sample_moment].update({'cpu_available' : (tmp_machines_df['cpu']).sum(), \n",
    "                                                         'mem_available' : (tmp_machines_df['mem']).sum()})\n",
    "            else:\n",
    "                samples_dicts[current_sample_moment].update({'cpu_available' : 0, \n",
    "                                                         'mem_available' : 0})\n",
    "            try:\n",
    "                current_sample_moment = next(sample_moments_iterator)\n",
    "            except StopIteration:\n",
    "                current_sample_moment = None\n",
    "        if event['machine_id'] == machine_id:\n",
    "            if event['event_type'] in [0, 2]:\n",
    "                machines_dict[event['machine_id']] = {'machine_id' : event['machine_id'], \n",
    "                                                      'cpu' : event['cpu'], 'mem' : event['mem']}\n",
    "            elif event['event_type'] in [1]:\n",
    "                del machines_dict[event['machine_id']]\n",
    "\n",
    "    if machines_df is not None and current_sample_moment is None:\n",
    "        break\n",
    "                    \n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "print samples_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(samples_df['time'], samples_df['cpu_requested'], label='cpu requested')\n",
    "ax.plot(samples_df['time'], samples_df['cpu_available'], label='cpu available')\n",
    "ax.plot(samples_df['time'], samples_df['mem_requested'], label='mem requested')\n",
    "ax.plot(samples_df['time'], samples_df['mem_available'], label='mem available')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\nInt64Index: 200 entries, 0 to 199\nData columns (total 9 columns):\ncpu_requested               200 non-null float64\ncpu_usage                   200 non-null float64\ndisk                        200 non-null float64\ndisk_io_time                200 non-null float64\ndisk_space                  200 non-null float64\nmem_requested               200 non-null float64\nmem_usage                   200 non-null float64\nnumber_of_task_requested    200 non-null int64\ntime                        200 non-null int64\ndtypes: float64(7), int64(2)None\n"
     ]
    }
   ],
   "source": [
    "# doc resource usage cho mot machine bat ky\n",
    "machine_id = 1268205\n",
    "task_usage_csv_colnames=['starttime', 'endtime', 'job_id', 'task_idx', 'machine_id', 'cpu_usage', 'mem_usage', \n",
    "                         'assigned_mem', 'unmapped_cache_usage', 'page_cache_usage', 'max_mem_usage', 'disk_io_time', \n",
    "                         'mean_local_disk_space', 'max_cpu_usage', 'max_disk_io_time', 'cpi', 'mai', 'sampling_portion', \n",
    "                         'agg_type', 'sampled_cpu_usage']\n",
    "\n",
    "for moment in samples_dicts:\n",
    "    samples_dicts[moment].update({'cpu_usage' : 0.0, 'mem_usage' : 0.0, 'disk_io_time': 0.0, 'disk_space': 0.0})\n",
    "    \n",
    "# %%time\n",
    "for fn in sorted(listdir('task_usage')):\n",
    "\n",
    "    fp = path.join('task_usage', fn)\n",
    "    task_usage_df = read_csv(fp, header=None, index_col=False, compression='gzip', names=task_usage_csv_colnames)\n",
    "    task_usage_df = task_usage_df[task_usage_df['machine_id']==machine_id]\n",
    "    \n",
    "    laststart = max(task_usage_df['starttime'])\n",
    "    firststart = min(task_usage_df['starttime'])\n",
    "    # if laststart > max(sample_moments) and laststart > snapshot_moment:\n",
    "    #     break\n",
    "    if firststart > max(sample_moments) and firststart > snapshot_moment:\n",
    "        break\n",
    "    for moment in samples_dicts:\n",
    "        task_usage_moment_df = task_usage_df[(task_usage_df['starttime'] <= moment) & \n",
    "                                             (moment <= task_usage_df['endtime'])]\n",
    "        # print task_usage_moment_df.info()\n",
    "        samples_dicts[moment]['cpu_usage'] += sum(task_usage_moment_df['cpu_usage'])\n",
    "        samples_dicts[moment]['mem_usage'] += sum(task_usage_moment_df['mem_usage'])\n",
    "        samples_dicts[moment]['disk_io_time'] += sum(task_usage_moment_df['disk_io_time'])\n",
    "        samples_dicts[moment]['disk_space'] += sum(task_usage_moment_df['mean_local_disk_space'])\n",
    "\n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "print samples_df.info()\n",
    "\n",
    "samples_df.to_csv(path_or_buf='results/random_sampling_machineid_'+str(machine_id)+'.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "# ax.plot(samples_df['time'], samples_df['cpu_requested'], label='cpu requested')\n",
    "# ax.plot(samples_df['time'], samples_df['cpu_available'], label='cpu available')\n",
    "ax.plot(samples_df['time'], samples_df['cpu_usage'], label='cpu usage')\n",
    "plt.xlim(min(samples_df['time']), max(samples_df['time']))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = read_csv('results/random_sampling_machineid_'+str(machine_id)+'.csv',index_col=False)\n",
    "fig = plt.figure()\n",
    "ax = fig.add_subplot(111)\n",
    "ax.plot(df['time'], df['cpu_requested'], label='cpu requested')\n",
    "ax.plot(df['time'], df['mem_requested'], label='mem requested')\n",
    "ax.plot(df['time'], df['disk'], label='disk space requested')\n",
    "plt.xlim(min(df['time']), max(df['time']))\n",
    "plt.title('machine ID = ' +str(machine_id))\n",
    "plt.legend()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     4155527081\n1      329150663\n2     3938719206\n3      351618647\n4      431052910\n5      257348783\n6     5655258253\n7     3550322224\n8        1303745\n9     3894543095\n10     336025676\n11    3405236527\n12     431081448\n13      84899647\n14       1268205\n...\n450131           NaN\n450132    4802517408\n450133     258727769\n450134           NaN\n450135     258727769\n450136       2850406\n450137           NaN\n450138    4820204869\n450139    4820223869\n450140     273549095\n450141    1436472882\n450142           NaN\n450143    1436472882\n450144           NaN\n450145           NaN\nName: machine_id, Length: 450146, dtype: float64"
      ]
     },
     "execution_count": 106,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "(task_event_df['machine_id'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "200"
      ]
     },
     "execution_count": 6,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "len(sample_moments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          NaN\n1          NaN\n2     0.000424\n3     0.000424\n4          NaN\n5          NaN\n6          NaN\n7          NaN\n8          NaN\n9          NaN\n10         NaN\n11         NaN\n12         NaN\n13         NaN\n14         NaN\n...\n450131    0.000115\n450132    0.000011\n450133    0.000077\n450134    0.000077\n450135    0.000011\n450136    0.000038\n450137    0.000038\n450138    0.000386\n450139    0.000386\n450140    0.000386\n450141    0.000019\n450142    0.000019\n450143    0.000011\n450144    0.000215\n450145    0.000215\nName: disk, Length: 450146, dtype: float64"
      ]
     },
     "execution_count": 41,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "task_events_df['disk']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'task_events/part-00000-of-00500.csv.gz'"
      ]
     },
     "execution_count": 23,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "fp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "reading file machine_events/part-00000-of-00001.csv.gz\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-27-8e8d3c58272f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     35\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     36\u001b[0m         \u001b[1;32mwhile\u001b[0m \u001b[0mcurrent_sample_moment\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mNone\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mevent\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m'time'\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m>\u001b[0m \u001b[0mcurrent_sample_moment\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 37\u001b[1;33m             \u001b[0mtmp_machines_df\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mDataFrame\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mmachines_dict\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     38\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mtmp_machines_df\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mempty\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     39\u001b[0m                 samples_dicts[current_sample_moment] = ({'time' : current_sample_moment,\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[0;32m    245\u001b[0m                         \u001b[0mindex\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_default_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    246\u001b[0m                     mgr = _arrays_to_mgr(arrays, columns, index, columns,\n\u001b[1;32m--> 247\u001b[1;33m                                          dtype=dtype)\n\u001b[0m\u001b[0;32m    248\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    249\u001b[0m                     mgr = self._init_ndarray(data, index, columns, dtype=dtype,\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/frame.pyc\u001b[0m in \u001b[0;36m_arrays_to_mgr\u001b[1;34m(arrays, arr_names, index, columns, dtype)\u001b[0m\n\u001b[0;32m   4471\u001b[0m     \u001b[0maxes\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[0m_ensure_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0m_ensure_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4472\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 4473\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mcreate_block_manager_from_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0marr_names\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   4474\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   4475\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mcreate_block_manager_from_arrays\u001b[1;34m(arrays, names, axes)\u001b[0m\n\u001b[0;32m   3753\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mcreate_block_manager_from_arrays\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3754\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3755\u001b[1;33m         \u001b[0mblocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mform_blocks\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrays\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mnames\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3756\u001b[0m         \u001b[0mmgr\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mBlockManager\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mblocks\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxes\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3757\u001b[0m         \u001b[0mmgr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_consolidate_inplace\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36mform_blocks\u001b[1;34m(arrays, names, axes)\u001b[0m\n\u001b[0;32m   3830\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3831\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint_items\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3832\u001b[1;33m         \u001b[0mint_blocks\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_multi_blockify\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint_items\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mitems\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mis_unique\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3833\u001b[0m         \u001b[0mblocks\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mint_blocks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3834\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36m_multi_blockify\u001b[1;34m(tuples, ref_items, dtype, is_unique)\u001b[0m\n\u001b[0;32m   3893\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3894\u001b[0m         block_items, values, placement = _stack_arrays(\n\u001b[1;32m-> 3895\u001b[1;33m             list(tup_block), ref_items, dtype)\n\u001b[0m\u001b[0;32m   3896\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mis_unique\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3897\u001b[0m             \u001b[0mplacement\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/internals.pyc\u001b[0m in \u001b[0;36m_stack_arrays\u001b[1;34m(tuples, ref_items, dtype)\u001b[0m\n\u001b[0;32m   3948\u001b[0m     \u001b[1;31m# index may box values\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3949\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mref_items\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mis_unique\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 3950\u001b[1;33m         \u001b[0mitems\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mref_items\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mref_items\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0misin\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnames\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   3951\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   3952\u001b[0m         \u001b[1;31m# a mi\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/index.pyc\u001b[0m in \u001b[0;36m__getitem__\u001b[1;34m(self, key)\u001b[0m\n\u001b[0;32m    619\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0marr_idx\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    620\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 621\u001b[1;33m             \u001b[1;32mif\u001b[0m \u001b[0mcom\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_is_bool_indexer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    622\u001b[0m                 \u001b[0mkey\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    623\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m/usr/lib/python2.7/dist-packages/pandas/core/common.pyc\u001b[0m in \u001b[0;36m_is_bool_indexer\u001b[1;34m(key)\u001b[0m\n\u001b[0;32m   1752\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1753\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mkey\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbool_\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1754\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1755\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1756\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#lay thong tin CPU, RAM available cua machine\n",
    "from pandas import read_csv\n",
    "from os import path\n",
    "import sys\n",
    "from os import listdir, chdir\n",
    "from collections import OrderedDict\n",
    "from pandas import DataFrame\n",
    "from random import randint, sample, seed\n",
    "\n",
    "data_directory = 'machine_events'\n",
    "results_directory = 'results'\n",
    "\n",
    "interval = 1\n",
    "\n",
    "maxtime_stamp = 2506199602822\n",
    "\n",
    "seed(83)\n",
    "sample_moments = range(0,maxtime_stamp,1000000* (int)(interval)) # lay mau theo giay\n",
    "# sample_moments = sorted(sample(xrange(maxtime_stamp), 200))\n",
    "\n",
    "machine_id = 1268205\n",
    "machines_dict = {}\n",
    "samples_dicts = OrderedDict([])\n",
    "sample_moments_iterator = iter(sample_moments)\n",
    "current_sample_moment = next(sample_moments_iterator)\n",
    "\n",
    "machine_events_csv_colnames=['time', 'machine_id', 'event_type', 'platform_id', 'cpu', 'mem']\n",
    "\n",
    "# %%time\n",
    "for fn in sorted(listdir(data_directory)):\n",
    "    fp = path.join(data_directory, fn)\n",
    "    machine_events_df = read_csv(fp, header=None, index_col=False, compression='gzip', names=machine_events_csv_colnames)\n",
    "    print 'reading file '+fp\n",
    "    for index, event in machine_events_df.iterrows():\n",
    "\n",
    "        while current_sample_moment is not None and event['time'] > current_sample_moment:\n",
    "            tmp_machines_df = DataFrame(machines_dict.values())\n",
    "            if not tmp_machines_df.empty:\n",
    "                samples_dicts[current_sample_moment] = ({'time' : current_sample_moment,\n",
    "                                                         'cpu_available' : (tmp_machines_df['cpu']).sum(),\n",
    "                                                         'mem_available' : (tmp_machines_df['mem']).sum()})\n",
    "            else:\n",
    "                samples_dicts[current_sample_moment] = ({'cpu_available' : 0,\n",
    "                                                         'mem_available' : 0})\n",
    "            try:\n",
    "                current_sample_moment = next(sample_moments_iterator)\n",
    "            except StopIteration:\n",
    "                current_sample_moment = None\n",
    "        if event['machine_id'] == machine_id:\n",
    "            if event['event_type'] in [0, 2]:\n",
    "                machines_dict[event['machine_id']] = {'machine_id' : event['machine_id'],\n",
    "                                                      'cpu' : event['cpu'], 'mem' : event['mem']}\n",
    "            elif event['event_type'] in [1]:\n",
    "                print 'co bi tat'\n",
    "                del machines_dict[event['machine_id']]\n",
    "\n",
    "\n",
    "    if current_sample_moment is None:\n",
    "        break\n",
    "\n",
    "samples_df = DataFrame(samples_dicts.values())\n",
    "print samples_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2506135493517"
      ]
     },
     "execution_count": 16,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "event['time']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25303000000"
      ]
     },
     "execution_count": 17,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "current_sample_moment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{1268205: {'cpu': 0.5, 'machine_id': 1268205, 'mem': 0.4995}}"
      ]
     },
     "execution_count": 18,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "machines_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25305000000"
      ]
     },
     "execution_count": 20,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "next(sample_moments_iterator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2506199000000"
      ]
     },
     "execution_count": 24,
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "max(sample_moments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cpu_available</th>\n",
       "      <th>mem_available</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>        0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  1000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  2000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  3000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  4000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  5000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  6000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  7000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  8000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9 </th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td>  9000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 10000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 11000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 12000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 13000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 14000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 15000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 16000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 17000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 18000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 19000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 20000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 21000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 22000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 23000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 24000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 25000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 26000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 27000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 28000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 29000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 30000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 31000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 32000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 33000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 34000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 35000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 36000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 37000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 38000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 39000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 40000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 41000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 42000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 43000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 44000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 45000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 46000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 47000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 48000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 49000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 50000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 51000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 52000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 53000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 54000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 55000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 56000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 57000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 58000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td> 0.5</td>\n",
       "      <td> 0.4995</td>\n",
       "      <td> 59000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25303 rows × 3 columns</p>\n",
       "</div>"
      ]
     },
     "output_type": "execute_result",
     "metadata": {}
    }
   ],
   "source": [
    "samples_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {},
 "nbformat": 4,
 "nbformat_minor": 0
}