Handy COSMOS ACS Functions.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Part 1: Stamp Phase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### General function definitions to be used for unlensed/lensed TensorFlow classifyer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# Import neccessary modules\n",
    "import os\n",
    "import csv\n",
    "import subprocess\n",
    "import numpy as np\n",
    "import random\n",
    "from astropy.io import fits\n",
    "from astropy import wcs\n",
    "from astropy import coordinates\n",
    "from astropy.nddata import Cutout2D\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def read_csv(csv_dir, output_format, verbose=True):\n",
    "    '''Returns a list o rows if output_format='list of rows' or a list of columns if output_format='list of columns'.'''\n",
    "    \n",
    "    with open(csv_dir, 'r') as f:\n",
    "        reader = csv.reader(f)\n",
    "        rows_of_data = list(reader)\n",
    "    \n",
    "    if output_format == 'list of rows':\n",
    "        return rows_of_data\n",
    "    \n",
    "    elif output_format == 'list of columns':\n",
    "        columns_of_data = list(map(list, zip(*rows_of_data)))\n",
    "        return columns_of_data\n",
    "    \n",
    "    else: print('Milad: Invalid output_format in read_csv(...).')\n",
    "    \n",
    "    if verbose:\n",
    "        print(csv_dir)\n",
    "\n",
    "def write_csv(data, csv_dir, data_format, verbose=True):\n",
    "    '''If data_format='list of rows' saves each element of data as a line to a csv file.\n",
    "    If data_format='list of columns', saves each element of data to a column.'''\n",
    "    \n",
    "    if data_format == 'list of rows':\n",
    "        lines = data\n",
    "            \n",
    "    elif data_format == 'list of columns':\n",
    "        lines = list(map(list, zip(*data)))\n",
    "        \n",
    "    else:\n",
    "        print('Milad: Invalid output_format in write_csv(...).')\n",
    "        return None\n",
    "        \n",
    "    with open(csv_dir, 'w') as f:\n",
    "        writer = csv.writer(f)\n",
    "        writer.writerows(lines)\n",
    "    \n",
    "    if verbose:\n",
    "        print(csv_dir)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def txt_to_csv(input_txt_dir, output_csv_dir=None):\n",
    "    if output_csv_dir is None:\n",
    "        output_csv_dir = input_txt_dir[:-4] + '.csv'\n",
    "    \n",
    "    data = np.loadtxt(input_txt_dir, comments='#')\n",
    "    np.set_printoptions(suppress=True)\n",
    "    np.savetxt(output_csv_dir, data, fmt='%f', delimiter=',')\n",
    "    \n",
    "    print('CSV created at:\\n' + output_csv_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def Map(f, item, level=1):\n",
    "    if level == 0:\n",
    "        return f(item)\n",
    "    else:\n",
    "        return [map_level(f, i, level - 1) for i in item]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_center_RA_Dec(input_fits_dir, x_pixel, y_pixel, RHS_num_digits = 6):\n",
    "    '''Receives input_fits_dir, pixel coordinates (e.g. 100) and returns Ra and Dec (in degrees) of the pixel at x_pix = 100, y_pix = 100 pixel.\n",
    "    Output Example: ('150.4403', '1.7549')'''\n",
    "\n",
    "    originalFile = fits.open(input_fits_dir) #importing input file\n",
    "\n",
    "    # Extract coord info of the original fits file\n",
    "    original_coords = wcs.WCS(originalFile[0].header) # Extract coords of the old FITS\n",
    "    \n",
    "    ra, dec = original_coords.wcs_pix2world(x_pixel, y_pixel, 1)\n",
    "    ra, dec = ra.tolist(), dec.tolist()\n",
    "    \n",
    "    ra  = (\"{0:.\" + str(RHS_num_digits) + \"f}\").format(ra)\n",
    "    dec = (\"{0:.\" + str(RHS_num_digits) + \"f}\").format(dec)\n",
    "    \n",
    "    return ra, dec\n",
    "\n",
    "# file_dirs = ['/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0001_150.0525000_+2.3375000_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0002_150.0579167_+2.3802778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0003_150.0766667_+2.6458333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0004_150.1591667_+2.6925000_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0005_150.1983333_+1.8397222_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0006_150.2050000_+1.8577778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0007_150.2108333_+2.8169444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0008_150.2362500_+2.2072222_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0009_150.3520833_+1.8558333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0010_150.5466667_+2.1941667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0011_150.5700000_+2.4986111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0012_150.6145833_+2.0808333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0013_150.7250000_+2.2416667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0015_149.4941667_+2.2569444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0016_149.7375000_+1.9969444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0017_149.8112500_+2.2052778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0018_149.8404167_+2.1105556_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0019_149.9220833_+2.6077778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0020_149.9491667_+2.7977778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0021_150.0404167_+2.4152778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0022_150.1191667_+2.3219444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0023_150.1233333_+2.6716667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0024_150.1962500_+2.4919444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0025_150.2116667_+2.0658333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0026_150.2320833_+1.6391667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0027_150.2362500_+2.3516667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0028_150.2704167_+2.3461111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0029_150.2691667_+2.4169444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0030_150.2720833_+2.7586111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0031_150.2829167_+2.0925000_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0032_150.2833333_+1.9350000_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0033_150.3341667_+1.7641667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0034_150.4504167_+2.3902778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0035_150.5354167_+2.2394444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0036_150.5841667_+2.3930556_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0037_150.5879167_+2.5777778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0038_150.6500000_+2.8019444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0039_150.7016667_+2.2394444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0040_149.4500000_+1.9233333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0041_149.4612500_+1.9386111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0042_149.4670833_+2.3491667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0043_149.4754167_+1.9977778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0044_149.5233333_+2.0702778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0045_149.5283333_+1.9691667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0046_149.5891667_+1.7436111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0047_149.6245833_+1.6261111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0048_149.6291667_+1.7255556_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0049_149.6725000_+2.7794444_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0050_149.7162500_+2.3036111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0051_149.7337500_+2.7986111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0052_149.7770833_+2.7566667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0053_149.8308333_+1.6480556_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0054_149.8529167_+2.1477778_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0055_149.8575000_+2.5108333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0056_149.8745833_+2.2311111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0057_149.8708333_+1.7647222_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0058_149.8795833_+2.0413889_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0059_149.8833333_+2.1716667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0060_149.9029167_+2.6058333_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0061_149.9129167_+2.5122222_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0062_149.9183333_+2.5480556_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0063_149.9166667_+2.8366667_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0064_149.9279167_+2.4747222_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0065_149.9295833_+2.4711111_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0066_149.9625000_+2.2100000_acs_I_mosaic_30mas_sci.fits','/home/USER/Desktop/LensFlow/Faure_Lenses/acs_mosaic_2.0/0067_149.9987500_+2.0633333_acs_I_mosaic_30mas_sci.fits']\n",
    "\n",
    "# newLensRADec = \"{{\"\n",
    "\n",
    "# for file_dir in file_dirs:\n",
    "#     ra, dec = get_center_RA_Dec(file_dir, 250, 250);\n",
    "#     newLensRADec += \"\\\"\" + str(ra) + \"\\\",\\\"\" + str(dec) + \"\\\"},\\n{\"\n",
    "\n",
    "# print(newLensRADec)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def single_crop(input_fits_dir, xCoord, yCoord, cutout_shape=(200, 200), output_fits_dir='/home/USER/Desktop/cutout.fits'):\n",
    "    '''Creates a cutout with cutout_shape of the input_fits_dir around xCoord and yCoord and saves it to the Desktop.'''\n",
    "    \n",
    "    originalFile = fits.open(input_fits_dir) #importing input file\n",
    "\n",
    "    xMax = originalFile[0].data.shape[0] # x position of last pixel\n",
    "    yMax = originalFile[0].data.shape[1] # y position of last pixel\n",
    "\n",
    "    # Extract coord info of the original fits file\n",
    "    original_coords = wcs.WCS(originalFile[0].header) # Extract coords of the old FITS\n",
    "\n",
    "    # Crop\n",
    "    cutout          = Cutout2D(originalFile[0].data, (xCoord, yCoord), cutout_shape, wcs=original_coords)\n",
    "\n",
    "    # Create a new fits object and add cutout data\n",
    "    new_file        = fits.PrimaryHDU()\n",
    "    new_file.data   = cutout.data\n",
    "\n",
    "    # Add coord info to the header\n",
    "    new_coords      = cutout.wcs\n",
    "    new_file.header.update(new_coords.to_header()) # Update new FITS coords\n",
    "    \n",
    "    # Save the new fits file\n",
    "    new_file.writeto(output_fits_dir) #exporting output file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_raws(fits_tile_dir, cat_name='cat.csv', output_subfolder_name='raws', cutout_shape=(200, 200), verbose=False):\n",
    "    '''Note: folder_dir must contain a fits file to be cropped and csv file containing the coords to be cropped.\n",
    "    Reads the coords in cat.csv file at folder_dir which was generated by \"SExtract\" and \"catTXT_to_catCSV\" functions.\n",
    "    Cuts a cutout_shape box around those coordinates from the fits file specified by fits_name.\n",
    "    Saves it to a folder called \"raws/\".\n",
    "    Remove the fits files that don't match cutout_shape, since some sources are too close to the edge of the original fits file.'''\n",
    "\n",
    "    folder_dir = os.path.dirname(fits_tile_dir) + '/'\n",
    "    fits_name  = os.path.basename(fits_tile_dir)\n",
    "    \n",
    "    # CSV catalog dir\n",
    "    cat_dir = folder_dir + cat_name # <<< set dir\n",
    "\n",
    "    # FITS file dir to be cropped\n",
    "    inputFolderDirectory  = folder_dir\n",
    "    inputName             = fits_name # .fits ext must be included\n",
    "    # dir of cropped FITS files\n",
    "    outputFolderDirectory = inputFolderDirectory + output_subfolder_name + '/'\n",
    "    outputName            = inputName[:-5] + '_' + output_subfolder_name\n",
    "    outputFileExtention   = '.fits'\n",
    "\n",
    "\n",
    "    # Read csv lines:\n",
    "    with open(cat_dir, 'r') as file:\n",
    "        reader = csv.reader(file)\n",
    "        lines = list(reader)\n",
    "\n",
    "    # Convert the lines into a list of integers:\n",
    "    list_cat = []\n",
    "\n",
    "    for row in range(len(lines)):\n",
    "        list_cat.append([int(float(lines[row][0])),int(float(lines[row][1]))])\n",
    "\n",
    "    print('Running this program will take about ' + str(0.22*len(list_cat)//60) + ' + ' + str(0.054*len(list_cat)//60) + ' minutes.')\n",
    "\n",
    "\n",
    "    originalFile = fits.open(inputFolderDirectory + inputName) #importing input file\n",
    "\n",
    "    xMax = originalFile[0].data.shape[0] # x position of last pixel\n",
    "    yMax = originalFile[0].data.shape[1] # y position of last pixel\n",
    "\n",
    "\n",
    "    for i in range(len(list_cat)):\n",
    "\n",
    "        if not os.path.exists(outputFolderDirectory): # checks if the output directory exists\n",
    "            os.makedirs(outputFolderDirectory) # if not, it creates it\n",
    "\n",
    "        xCoord = list_cat[i][0]\n",
    "        yCoord = list_cat[i][1]\n",
    "        \n",
    "        if verbose == True:\n",
    "            print(str(i) + '. x: ' + str(xCoord) + ', y: ' + str(yCoord))\n",
    "\n",
    "        # Extract coord info of the original fits file\n",
    "        original_coords = wcs.WCS(originalFile[0].header) # Extract coords of the old FITS\n",
    "        \n",
    "        # Crop\n",
    "        cutout          = Cutout2D(originalFile[0].data, (xCoord, yCoord), cutout_shape, wcs=original_coords)\n",
    "        \n",
    "        # Create a new fits object and add cutout data\n",
    "        new_file        = fits.PrimaryHDU()\n",
    "        new_file.data   = cutout.data\n",
    "        \n",
    "        # Add coord info to the header\n",
    "        new_coords      = cutout.wcs\n",
    "        new_file.header.update(new_coords.to_header()) # Update new FITS coords\n",
    "        \n",
    "        # Save the new fits file\n",
    "        new_file.writeto(outputFolderDirectory + outputName + \"_i\" + str(i) + \"_x\" + str(xCoord) + \"_y\" + str(yCoord) + outputFileExtention) #exporting output file\n",
    "\n",
    "\n",
    "    # Delete oddly shaped fits files\n",
    "    scan_this_dir = outputFolderDirectory\n",
    "\n",
    "    for file_name in os.listdir(scan_this_dir):\n",
    "        if fits.getdata(scan_this_dir + file_name).shape != cutout_shape:\n",
    "            print('deleted: ' + scan_this_dir + file_name)\n",
    "            os.remove(scan_this_dir + file_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def save_fits(data_array, save_to, original_fits_dir=None, match=True, replace_duplicates=False):\n",
    "    '''Takes a numpy array and saves it to save_to directory.\n",
    "    Provided the original fits file with the same dimentions, it will match the header of the new fits file.\n",
    "    Last modification date: Mar. 2, 2017'''\n",
    "        \n",
    "    # Create a fits object\n",
    "    newFile = fits.PrimaryHDU()\n",
    "    \n",
    "    # Transfer image to the new fits file\n",
    "    newFile.data = data_array\n",
    "    \n",
    "    # If conditions are met, match the new header witht the original one\n",
    "    if original_fits_dir is not None and match is True:\n",
    "        # Extract original fits header\n",
    "        original_fits_header = fits.getheader(original_fits_dir)\n",
    "        # Transfer header and coord info to the new fits file\n",
    "        newFile.header = original_fits_header\n",
    "   \n",
    "    # Move files with the samve directory as destination to Ubuntu's trashcan\n",
    "    if replace_duplicates is True and os.path.exists(save_to):\n",
    "        subprocess.check_call(['gvfs-trash', save_to])\n",
    "    # Save to hard drive\n",
    "    newFile.writeto(save_to)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def list_and_label_files(folder_dir, file_label, valid_ext='.fits'):\n",
    "    '''Creates a CSV file with column A, B, C as folder_dir, file_name with extension, file_label respectively.\n",
    "    The CSV file will be save at the directory containing the searched folder.\n",
    "    Can be used to create a CSV list of files for \"raws\", or \"lenses\".\n",
    "    I've been only using the following labels: 'unlensed', 'lensed', 'unknown' '''\n",
    "    \n",
    "    csv_dir    = folder_dir[:-1] + '.csv'\n",
    "\n",
    "    data = [[],[],[]] #[File Directories, File Names, Labels]\n",
    "\n",
    "    for file_name in os.listdir(folder_dir):\n",
    "        if file_name[-(len(valid_ext)):] == valid_ext:\n",
    "            data[0].append(folder_dir)\n",
    "            data[1].append(file_name)\n",
    "            data[2].append(file_label)\n",
    "\n",
    "    # Save data to a CSV file\n",
    "    write_csv(data, csv_dir, data_format='list of columns')\n",
    "    \n",
    "    # Give a warning if file_label is not either lens or unlensed\n",
    "    if not(file_label=='lensed' or file_label=='unlensed'):\n",
    "        print('Milad: Check your file_label for list_and_label_files.')\n",
    "        \n",
    "    return csv_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def merge_raws_csv_files():\n",
    "    '''Merges the raws.csv files for each tile and stores them in the same directory as this notebook.'''\n",
    "    \n",
    "    all_raws_lines = []\n",
    "\n",
    "    for i in rng:\n",
    "        csv_dir = folder_dirs[i] + 'raws.csv'\n",
    "        one_tile_raws_lines = read_csv(csv_dir, output_format='list of rows')\n",
    "        all_raws_lines.extend(one_tile_raws_lines)\n",
    "    \n",
    "    merged_csv_dir = '/home/USER/Desktop/merged_raws.csv'\n",
    "    write_csv(data=all_raws_lines, csv_dir=merged_csv_dir, data_format='list of rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge_two_csv_files(csv_dir_1, csv_dir_2, output_csv_dir, shuffle_lines=True, match_len=False):\n",
    "    '''Combines the csv files at csv_dir_1 and csv_dir_2 and saves it at output_csv_dir.\n",
    "    Shuffles lines if shuffle_lines is true.\n",
    "    If match_len is true, it will only merge the first n_min lines, where n_min is the number of lines in the shorter csv.'''\n",
    "    \n",
    "    all_lines = []\n",
    "    \n",
    "    lines_1 = read_csv(csv_dir_1, output_format='list of rows')\n",
    "    n_1 = len(lines_1)\n",
    "    \n",
    "    \n",
    "    lines_2 = read_csv(csv_dir_2, output_format='list of rows')\n",
    "    n_2 = len(lines_2)\n",
    "    \n",
    "    \n",
    "    n_min = min(n_1, n_2)\n",
    "    \n",
    "    if match_len is True:\n",
    "        if shuffle_lines is True:\n",
    "            random.shuffle(lines_1)\n",
    "            random.shuffle(lines_2)\n",
    "        all_lines.extend(lines_1[:n_min])\n",
    "        all_lines.extend(lines_2[:n_min])\n",
    "    else:\n",
    "        all_lines.extend(lines_1)\n",
    "        all_lines.extend(lines_2)\n",
    "        \n",
    "    if shuffle_lines is True:\n",
    "        random.shuffle(all_lines)\n",
    "        \n",
    "    write_csv(data=all_lines, csv_dir=output_csv_dir, data_format='list of rows')\n",
    "    \n",
    "    return output_csv_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_sex_files(dst_dir, default_SExtractor_files_dir):\n",
    "    '''Copies default .conv, .nnw, and .psf from default_SExtractor_files_dir to dst_dir and\n",
    "    writes .sex and .param files specified here.'''\n",
    "    \n",
    "    sex_content = '''# Non-default configuration file for SExtractor 2.5.0\n",
    "    # EB 2006-07-14\n",
    "    #\n",
    "\n",
    "    #-------------------------------- Catalog ------------------------------------\n",
    "\n",
    "    CATALOG_NAME     cat.txt        # name of the output catalog\n",
    "    CATALOG_TYPE     ASCII_HEAD     # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT,\n",
    "                                    # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC\n",
    "    PARAMETERS_NAME  LF_01.param  # name of the file containing catalog contents\n",
    "\n",
    "    #------------------------------- Extraction ----------------------------------\n",
    "\n",
    "    DETECT_TYPE      CCD            # CCD (linear) or PHOTO (with gamma correction)\n",
    "    DETECT_MINAREA   200            # minimum number of pixels above threshold\n",
    "    DETECT_THRESH    1.5            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2\n",
    "    ANALYSIS_THRESH  2            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2\n",
    "\n",
    "    FILTER           Y              # apply filter for detection (Y or N)?\n",
    "    FILTER_NAME      default.conv   # name of the file containing the filter\n",
    "\n",
    "    DEBLEND_NTHRESH  32             # Number of deblending sub-thresholds\n",
    "    DEBLEND_MINCONT  0.005          # Minimum contrast parameter for deblending\n",
    "\n",
    "    CLEAN            Y              # Clean spurious detections? (Y or N)?\n",
    "    CLEAN_PARAM      1.0            # Cleaning efficiency\n",
    "\n",
    "    MASK_TYPE        CORRECT        # type of detection MASKing: can be one of\n",
    "                                    # NONE, BLANK or CORRECT\n",
    "\n",
    "    WEIGHT_TYPE      NONE\n",
    "\n",
    "    #------------------------------ Photometry -----------------------------------\n",
    "\n",
    "    #PHOT_APERTURES   5              # MAG_APER aperture diameter(s) in pixels\n",
    "    #PHOT_AUTOPARAMS  2.5, 3.5       # MAG_AUTO parameters: <Kron_fact>,<min_radius>\n",
    "    #PHOT_PETROPARAMS 2.0, 3.5       # MAG_PETRO parameters: <Petrosian_fact>,\n",
    "                                    # <min_radius>\n",
    "\n",
    "    #SATUR_LEVEL      50000.0        # level (in ADUs) at which arises saturation\n",
    "\n",
    "    #MAG_ZEROPOINT    0.0            # magnitude zero-point\n",
    "    #MAG_GAMMA        4.0            # gamma of emulsion (for photographic scans)\n",
    "    #GAIN             0.0            # detector gain in e-/ADU\n",
    "    #PIXEL_SCALE      1.0            # size of pixel in arcsec (0=use FITS WCS info)\n",
    "\n",
    "    #------------------------- Star/Galaxy Separation ----------------------------\n",
    "\n",
    "    SEEING_FWHM      1.2            # stellar FWHM in arcsec\n",
    "    STARNNW_NAME     default.nnw    # Neural-Network_Weight table filename\n",
    "\n",
    "    #------------------------------ Background -----------------------------------\n",
    "\n",
    "    BACK_SIZE        64             # Background mesh: <size> or <width>,<height>\n",
    "    BACK_FILTERSIZE  3              # Background filter: <size> or <width>,<height>\n",
    "\n",
    "    BACKPHOTO_TYPE   GLOBAL         # can be GLOBAL or LOCAL\n",
    "\n",
    "    #------------------------------ Check Image ----------------------------------\n",
    "\n",
    "    CHECKIMAGE_TYPE  MODELS, -MODELS, -BACKGROUND          # can be NONE, BACKGROUND, BACKGROUND_RMS,\n",
    "                                    # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND,\n",
    "                                    # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION,\n",
    "                                    # or APERTURES\n",
    "    CHECKIMAGE_NAME  prof.fits,subprof.fits,orig.fits     # Filename for the check-image\n",
    "\n",
    "    #--------------------- Memory (change with caution!) -------------------------\n",
    "\n",
    "    MEMORY_OBJSTACK  3000           # number of objects in stack\n",
    "    MEMORY_PIXSTACK  300000         # number of pixels in stack\n",
    "    MEMORY_BUFSIZE   1024           # number of lines in buffer\n",
    "\n",
    "    #----------------------------- Miscellaneous ---------------------------------\n",
    "\n",
    "    VERBOSE_TYPE     NORMAL         # can be QUIET, NORMAL or FULL\n",
    "    WRITE_XML        N              # Write XML file (Y/N)?\n",
    "    XML_NAME         sex.xml        # Filename for XML output\n",
    "    '''\n",
    "    #-----------------------------------------------------------------------------------------\n",
    "\n",
    "    param_content = '''# Catalog parameters \n",
    "    X_IMAGE\n",
    "    Y_IMAGE\n",
    "    A_IMAGE\n",
    "    B_IMAGE\n",
    "    FLUX_PSF\n",
    "    CLASS_STAR\n",
    "    '''\n",
    "\n",
    "    #-----------------------------------------------------------------------------------------\n",
    "\n",
    "    with open(dst_dir + 'LF_01.sex', 'w') as f:\n",
    "        f.write(sex_content)\n",
    "\n",
    "    with open(dst_dir + 'LF_01.param', 'w') as f:\n",
    "        f.write(param_content)\n",
    "\n",
    "    conv_src_dir = default_SExtractor_files_dir + 'default.conv'\n",
    "    nnw_src_dir  = default_SExtractor_files_dir + 'default.nnw'\n",
    "    psf_src_dir  = default_SExtractor_files_dir + 'default.psf'\n",
    "\n",
    "    subprocess.check_call(['cp', conv_src_dir, dst_dir + 'default.conv'])\n",
    "    subprocess.check_call(['cp', nnw_src_dir,  dst_dir + 'default.nnw'])\n",
    "    subprocess.check_call(['cp', psf_src_dir,  dst_dir + 'default.psf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def SExtract(fits_tile_dir):\n",
    "    '''Note: Make sure default.conv, default.param, default.psf, default.sex and the fits file to be source extracted are present at folder_dir.'''\n",
    "        \n",
    "    folder_dir = os.path.dirname(fits_tile_dir) + '/'\n",
    "    fits_name  = os.path.basename(fits_tile_dir)\n",
    "    \n",
    "    # Generate the necessary SExtractor files\n",
    "    generate_sex_files(dst_dir=folder_dir,\n",
    "                       default_SExtractor_files_dir='/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/SE_Files/')\n",
    "    \n",
    "    # Run SExtractor\n",
    "    !cd $folder_dir && pwd && /usr/bin/sextractor -c LF_01.sex $fits_name\n",
    "    \n",
    "    # Move the orig.fits file generated by SExtractor to Ubuntu's Trash\n",
    "    try:\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'orig.fits'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'default.conv'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'default.nnw'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'default.psf'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'LF_01.sex'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'LF_01.param'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'prof.fits'])\n",
    "        subprocess.check_call(['gvfs-trash', folder_dir + 'subprof.fits'])\n",
    "    except:\n",
    "        print('Milad: error?')\n",
    "        \n",
    "    # Convert cat.txt to cat.csv\n",
    "    txt_to_csv(folder_dir + 'cat.txt')\n",
    "    subprocess.check_call(['gvfs-trash', folder_dir + 'cat.txt'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Specific to COSMOS_ACS databse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def COSMOS_ACS_file_handler(COSMOS_folder_dir):\n",
    "    '''retruns fits_names, folder_dirs, fits_dirs for COSMOS_ACS data which can be found at http://irsa.ipac.caltech.edu/data/COSMOS/images/acs_mosaic_2.0/\n",
    "    The outputs can be used to auto download the data and auto run the functions in this notebook.'''\n",
    "    \n",
    "    # Create a list of integer indices for COSMOS ACS Mosaic Information\n",
    "    mother_name = 'COSMOS_ASC_'\n",
    "    int_indices = []\n",
    "    for i in [13,25,37,49,61,73,85,97,109]:\n",
    "        for j in range(0,9):\n",
    "            int_indices.append(i+j)\n",
    "\n",
    "    # Convert it to strings\n",
    "    str_indices = list(map(str,int_indices))\n",
    "\n",
    "    # Change to 'nnn' format\n",
    "    def nnn_format(index):\n",
    "        '''Changes 'n', 'nn' to 'nnn' format.\n",
    "        E.g. '23' becomes '023' '''\n",
    "        \n",
    "        if len(index) == 1:\n",
    "            return '00' + index\n",
    "        if len(index) == 2:\n",
    "            return '0' + index\n",
    "        if len(index) == 3:\n",
    "            return '' + index\n",
    "        if len(index) >  3:\n",
    "            return 'Milad: invalid format!'\n",
    "\n",
    "    str_indices = list(map(nnn_format, str_indices))\n",
    "\n",
    "    # Create a list of fits dirs\n",
    "    pwd = COSMOS_folder_dir\n",
    "\n",
    "    folder_dirs = []\n",
    "    fits_names  = []\n",
    "    fits_dirs   = []\n",
    "\n",
    "    for index in str_indices:\n",
    "        folder_name = mother_name + index + '/'\n",
    "        fits_name   = mother_name + index + '.fits'\n",
    "\n",
    "        fits_names.append(fits_name)\n",
    "        folder_dirs.append(pwd + folder_name)\n",
    "        fits_dirs.append(pwd + folder_name + fits_name)\n",
    "    return fits_names, folder_dirs, fits_dirs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Get directories and file names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# fits_names, folder_dirs, fits_dirs = ['COSMOS_ASC_.fits'],['/home/USER/Desktop/FITS_Data/COSMOS_ASC_/'],['/home/USER/Desktop/FITS_Data/COSMOS_ASC_/COSMOS_ASC_.fits']\n",
    "fits_names, folder_dirs, fits_dirs = COSMOS_ACS_file_handler('/media/USER/black_book/USER/COSMOS/')\n",
    "\n",
    "num_tiles = len(folder_dirs)\n",
    "num_tiles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "rng = range(num_tiles)\n",
    "for i in rng:\n",
    "    print(fits_names[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Run SExtractor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# for i in rng:\n",
    "#     SExtract(folder_dirs[i] + fits_names[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Create raws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# for i in rng:\n",
    "#     create_raws(folder_dirs[i] + fits_names[i])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Create a CSV list of fits files in raws folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# for i in rng:\n",
    "#     list_and_label_files(folder_dir=(folder_dirs[i] + 'raws/'), valid_ext='.fits', file_label='unlensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_069/COSMOS_ASC_069.fits')\n",
    "# list_and_label_files('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_069/raws/',file_label='unlensed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "#### Merge raws CSV files into one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# merge_raws_csv_files()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Part 2: Lens Phase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Create rings using LensTool:\n",
    "\n",
    "Open 'Auto ERing Creator v4' and after adjusting the parameters and file directories, run it. It will create lensed fits files along side with lenstool input files neccessary to generate them.\n",
    "\n",
    "Normalize rings:\n",
    "\n",
    "There is no need to normalize the rings since it would be normalized in the next section.\n",
    "\n",
    "Merge rings and raws to make lenses:\n",
    "\n",
    "Use FITS Merger v2 notebook to create lenses by adding a raw FITS file to a simulated ring.\n",
    "\n",
    "Clean the lenses:\n",
    "\n",
    "I have gone through all the lenses and have made a list of the bad lenses. Then, by using 'Lens Cleaner v1', I have copied the good lenses to a directory. Then I have renamed this directory to '1000_clean_lenses'.\n",
    "\n",
    "Create a CSV list of files for \"lenses\":"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Defining the neccessary functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def list_file_names(folder_dir, valid_ext='.fits', shuffle_names=True):\n",
    "    '''Returns a shuffled (if True) list of all the file names ending with '.fits' (if unchanged.) found in the specified folder_dir.'''\n",
    "    file_names = []\n",
    "    \n",
    "    for file_name in os.listdir(folder_dir):\n",
    "        if file_name[-len(valid_ext):] == valid_ext:\n",
    "            file_names.append(file_name)\n",
    "    \n",
    "    if shuffle_names is True:\n",
    "        random.shuffle(file_names)\n",
    "        \n",
    "    return file_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_fits_data(file_directory, method='raw'):\n",
    "    '''method: 'raw', 'divide by max', 'divide by sum'\n",
    "    This function will go to a given directory, extract fits image data and returns a normalized version of it.'''\n",
    "    \n",
    "    image_data     = fits.getdata(file_directory)\n",
    "    \n",
    "    if method == 'raw':\n",
    "        return image_data\n",
    "    \n",
    "    elif method == 'divide by max':     \n",
    "        # Divide by highest pixel value\n",
    "        image_data     = image_data/np.max(image_data)\n",
    "        \n",
    "        return image_data\n",
    "        \n",
    "    elif method == 'divide by sum':\n",
    "        #shifting pixel values up so the lowest one is zero\n",
    "        image_data     = image_data + max(-np.min(image_data),0)\n",
    "        #normalizing\n",
    "        image_data     = image_data/np.sum(image_data)\n",
    "        \n",
    "        return image_data\n",
    "    \n",
    "    elif method == 'norm':\n",
    "        image_data = image_data - np.mean(image_data)\n",
    "        image_data = image_data/np.std(image_data)\n",
    "        return image_data\n",
    "    \n",
    "    else:\n",
    "        print('Milad: Invalid method for get_fits_data(...).')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge_two_fits(dir_1, dir_2, coef_2, output_dir):\n",
    "    '''Gets the normalized image data of the two fits files specified at dir_1 and dir_2.\n",
    "    Multiplies the second one by coef_2 and adds it to the first one.\n",
    "    Saves the normalized result at output_dir.\n",
    "    Note: This will overwrite output_dir if it is already an existing file.'''\n",
    "    \n",
    "    image_1 = get_fits_data(dir_1, method='divide by max')\n",
    "    image_2 = get_fits_data(dir_2, method='divide by max')\n",
    "    \n",
    "    # If the dimension is right, add and normalize\n",
    "    if image_1.shape == image_2.shape:\n",
    "        image_3 = image_1 + coef_2 * image_2\n",
    "        sum3 = np.sum(image_3)\n",
    "        if sum3 > 0:\n",
    "            image_3 = image_3/np.sum(image_3)\n",
    "    else:\n",
    "        print('Milad: Different fits dimensions in merge_fits(...).')\n",
    "    \n",
    "    # Save to output_dir\n",
    "    save_fits(image_3, output_dir, replace_duplicates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def auto_merge(raws_folder_dir, ring_folder_dir, output_folder_dir, num_merges=None, coef_range=(0.5, 1.5)):\n",
    "    raw_names  = list_file_names(raws_folder_dir)\n",
    "    num_raws   = len(raw_names)\n",
    "    \n",
    "    ring_names = list_file_names(ring_folder_dir)\n",
    "    num_rings  = len(ring_names)\n",
    "    \n",
    "    # Checks if a folder with the same name exist; if so, delete it and then create one; if not create one.\n",
    "    if os.path.exists(output_folder_dir):\n",
    "        subprocess.call(['rm', '-r', output_folder_dir])\n",
    "        subprocess.call(['mkdir', output_folder_dir])\n",
    "    else:\n",
    "        subprocess.call(['mkdir', output_folder_dir])\n",
    "        \n",
    "    if num_merges is None:\n",
    "        num_merges = min(num_raws, num_rings)\n",
    "    \n",
    "    for i in range(num_merges):\n",
    "        # Randomize ring brightness\n",
    "        coef = random.uniform(*coef_range)\n",
    "        # Merge and save\n",
    "        merge_two_fits(dir_1=raws_folder_dir + raw_names[i], dir_2=ring_folder_dir + ring_names[i], coef_2=coef, output_dir=output_folder_dir + 'lens_' + str(i) + '.fits')\n",
    "    \n",
    "    !nautilus $output_folder_dir"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Merging raws and lenses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# auto_merge('COSMOS_ASC_055/raws/', 'COSMOS_ASC_055/rings/', 'COSMOS_ASC_055/lenses/')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Part 3: Primary Train Phase"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def create_test_and_train_csv_data_sets(raws_csv_dir, lenses_csv_dir, \n",
    "                                        train_csv_dir, test_csv_dir, \n",
    "                                        n_raws_train=2000, n_lenses_train=900):\n",
    "    '''Creates a testing data set and a training data set. These are csv files with with coloumn A, B, C being forlder directory, file name, file label respectively.\n",
    "\n",
    "    Note to Milad:\n",
    "    The following code goes to the raw data set csv file and takes the first n_train_raw.\n",
    "    Let's say the raw csv file has n_raw lines and lensed csv file has n_lensed lines.\n",
    "    Remove n_raws_train lines from the raw csv and n_lenses_train from the lensed csv, combine and store them in a csv file called train_01.csv.\n",
    "    Combine the remaining lines from raw csv and lensed csv and store them in a csv file called test_01.csv.\n",
    "    '''\n",
    "\n",
    "    # Read raws\n",
    "    raws_lines = read_csv(raws_csv_dir, output_format='list of rows')\n",
    "    random.shuffle(raws_lines)\n",
    "    n_raws = len(raws_lines)\n",
    "\n",
    "    # Read lenses\n",
    "    lenses_lines = read_csv(lenses_csv_dir, output_format='list of rows')\n",
    "    random.shuffle(lenses_lines)\n",
    "    n_lenses = len(lenses_lines)\n",
    "\n",
    "\n",
    "    # Find the maximum number of raws and lenses left \n",
    "    #  from the testing set for the training set\n",
    "    max_n_raws_test   = n_raws   - n_raws_train\n",
    "\n",
    "    max_n_lenses_test = n_lenses - n_lenses_train\n",
    "\n",
    "\n",
    "    # Here, we make sure there are equal number of raws and lenses\n",
    "\n",
    "    half_test_size = min(max_n_raws_test, max_n_lenses_test)\n",
    "    n_raws_test = half_test_size\n",
    "    n_lenses_test = half_test_size\n",
    "\n",
    "    #>>> Copy\n",
    "\n",
    "    # Copy raws   for training:\n",
    "    train_raws_lines = raws_lines[:n_raws_train]\n",
    "    # Copy lenses for training:\n",
    "    train_lenses_lines = lenses_lines[:n_lenses_train]\n",
    "\n",
    "    # Copy raws   for testing:\n",
    "    test_raws_lines  = raws_lines[-n_raws_test:]\n",
    "    # Copy lenses for testing:\n",
    "    test_lenses_lines  = lenses_lines[-n_lenses_test:]\n",
    "\n",
    "    #<<< Copy\n",
    "\n",
    "    #>>> Combine, shuffle and save trains\n",
    "    train_lines = []\n",
    "\n",
    "    train_lines.extend(train_raws_lines)\n",
    "    train_lines.extend(train_lenses_lines)\n",
    "\n",
    "    random.shuffle(train_lines)\n",
    "\n",
    "    write_csv(train_lines, train_csv_dir, data_format='list of rows')\n",
    "\n",
    "    #<<< Combine, shuffle and save trains\n",
    "\n",
    "\n",
    "    #>>> Combine, shuffle and save tests\n",
    "    test_lines = []\n",
    "\n",
    "    test_lines.extend(test_raws_lines)\n",
    "    test_lines.extend(test_lenses_lines)\n",
    "\n",
    "    random.shuffle(test_lines)\n",
    "\n",
    "    write_csv(test_lines, test_csv_dir, data_format='list of rows')\n",
    "\n",
    "    #<<< Combine, shuffle and save tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_test_and_train_csv_data_sets_v2(raws_csv_dir, lenses_csv_dir, \n",
    "                                        train_csv_dir, test_csv_dir, \n",
    "                                        n_half_test=100):\n",
    "    '''Creates a testing data set and a training data set. These are csv files with with coloumn A, B, C being forlder directory, file name, file label respectively.\n",
    "\n",
    "    Note to Milad:\n",
    "    The following code goes to the raw data set csv file and takes the first n_train_raw.\n",
    "    Let's say the raw csv file has n_raw lines and lensed csv file has n_lensed lines.\n",
    "    Remove n_raws_train lines from the raw csv and n_lenses_train from the lensed csv, combine and store them in a csv file called train_01.csv.\n",
    "    Combine the remaining lines from raw csv and lensed csv and store them in a csv file called test_01.csv.\n",
    "    '''\n",
    "\n",
    "    # Read raws\n",
    "    raws_lines = read_csv(raws_csv_dir, output_format='list of rows')\n",
    "    random.shuffle(raws_lines)\n",
    "    n_raws = len(raws_lines)\n",
    "\n",
    "    # Read lenses\n",
    "    lenses_lines = read_csv(lenses_csv_dir, output_format='list of rows')\n",
    "    random.shuffle(lenses_lines)\n",
    "\n",
    "    #>>> Copy\n",
    "\n",
    "    # Copy raws   for training:\n",
    "    test_raws_lines    = raws_lines[:n_half_test]\n",
    "    # Copy lenses for training:\n",
    "    test_lenses_lines  = lenses_lines[:n_half_test]\n",
    "\n",
    "    # Copy raws   for testing:\n",
    "    train_raws_lines   = raws_lines[n_half_test:]\n",
    "    # Copy lenses for testing:\n",
    "    train_lenses_lines = lenses_lines[n_half_test:]\n",
    "\n",
    "    #<<< Copy\n",
    "\n",
    "    \n",
    "    #>>> Combine, shuffle and save tests\n",
    "    test_lines = []\n",
    "\n",
    "    test_lines.extend(test_raws_lines)\n",
    "    test_lines.extend(test_lenses_lines)\n",
    "\n",
    "    random.shuffle(test_lines)\n",
    "\n",
    "    write_csv(test_lines, test_csv_dir, data_format='list of rows')\n",
    "\n",
    "    #<<< Combine, shuffle and save tests\n",
    "    \n",
    "    \n",
    "    #>>> Combine, shuffle and save trains\n",
    "    train_lines = []\n",
    "\n",
    "    train_lines.extend(train_raws_lines)\n",
    "    train_lines.extend(train_lenses_lines)\n",
    "\n",
    "    random.shuffle(train_lines)\n",
    "\n",
    "    write_csv(train_lines, train_csv_dir, data_format='list of rows')\n",
    "\n",
    "    #<<< Combine, shuffle and save trains"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Bunch of function I ran:\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/good_lenses/', file_label='lensed')\n",
    "# names = list_file_names('./COSMOS_ASC_055/good_lenses/', shuffle_names=False)\n",
    "\n",
    "# for name in names:\n",
    "#     transform_fits('./COSMOS_ASC_055/good_lenses/', name, './COSMOS_ASC_055/good_lenses_x8/')\n",
    "    \n",
    "# list_and_label_files('./COSMOS_ASC_055/good_lenses_x8/', file_label='lensed')\n",
    "\n",
    "# ----\n",
    "\n",
    "# for i in range(1, 10):\n",
    "#     fd = './COSMOS_ASC_055/raws/raw_000'+ str(i) + '.fits'\n",
    "#     save_fits(enhance_fits_2(fd), save_to='/home/USER/Desktop/abc_' + str(i) + '.fits', replace_duplicates=True)\n",
    "\n",
    "# save_fits(enhance_fits_2('./COSMOS_ASC_055/raws/raw_3589.fits'), save_to='/home/USER/Desktop/not_from_TF.fits', replace_duplicates=True)\n",
    "\n",
    "# os.path.exists('/home/USER/Desktop/abc_1.fits')\n",
    "\n",
    "# subprocess.check_call('gvfs-trash '/home/USER/Desktop/abc_1.fits'')\n",
    "\n",
    "# merge_two_fits('/home/USER/Desktop/abc_2.fits', '/home/USER/Desktop/abc_3.fits', 3, '/home/USER/Desktop/cab.fits')\n",
    "\n",
    "# transform_fits('/home/USER/Desktop/','abc_1.fits', '/home/USER/Desktop/')\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/raws/', file_label='unlensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create_test_and_train_csv_data_sets('COSMOS_ASC_055/raws/', 'COSMOS_ASC_055/lenses/', 'COSMOS_ASC_055/train.csv', 'COSMOS_ASC_055/test.csv', )\n",
    "# \n",
    "# list_and_label_files('COSMOS_ASC_055/lenses/', 'lensed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "### Train CNN\n",
    "\n",
    "#### On MNIST data\n",
    "\n",
    "To insure the CNN works properly, I have created FITS files of digits 0's and 1's from the test set since it is smaller and stored them at:\n",
    "\n",
    "    /home/USER/Desktop/Jupyter_Notebooks_16.01/FITS_Data/MNIST_Test/\n",
    "\n",
    "In the same directory, you can find the training set 'train_zero_one.csv'.\n",
    "\n",
    "This was done via 'MNIST to FITS v1.ipynb'\n",
    "\n",
    "#### On real data\n",
    "\n",
    "Under FITS Data Handling, set the correct directories for training and testing CSV files."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "## Part 4: Search Phase"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Go to "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "deletable": true,
    "editable": true
   },
   "source": [
    "Use the following function to view multiple fits files to further filter lens candidates."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def multi_ds9(csv_dir, indices=range(0, 100)):\n",
    "    '''Displays multiple fits files specified by indeces of csv_dir.\n",
    "    A, B columns of the specified csv must be folder directory and file name with extension respectively.\n",
    "    Can be used to view the most lens-like candidates generated by the neural network at results.csv.\n",
    "    Last Modification Date: Mar 03, 2017'''\n",
    "    \n",
    "    lines = read_csv(csv_dir, output_format='list of rows')\n",
    "    \n",
    "    selected_dirs = []\n",
    "\n",
    "    for i in indices:\n",
    "        selected_dirs.append(lines[i][0] + lines[i][1])\n",
    "    subprocess.call(\n",
    "        ['cd \"$1\" || exit; shift; exec ds9 \"$@\"', \"_\", '']+selected_dirs,\n",
    "        shell=True);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# multi_ds9('COSMOS_ASC_055/lenses_results.csv', range(0,100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true,
    "deletable": true,
    "editable": true
   },
   "source": [
    "# Part 5: Secondary Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "deletable": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "def transform_fits(input_folder_dir, input_name, output_folder_dir):\n",
    "    '''Opens the fits file at the given directory and applies all the unique transformations to it\n",
    "    which are saved at the same directory with the name of the transformatin added to the file name.\n",
    "    No transformation is also included.\n",
    "    The coordinate transformation is not handled in this function.\n",
    "    Last Modified on Mar. 15, 2017.'''\n",
    "    \n",
    "    def rotate_none(mtx):\n",
    "        '''Retruns the same numpy matrix.'''\n",
    "        return mtx\n",
    "\n",
    "    def rotate_once(mtx):\n",
    "        '''Rotates the given numpy matrix by 90 degree counterclockwise.'''\n",
    "        return np.rot90(mtx)\n",
    "\n",
    "    def rotate_twice(mtx):\n",
    "        '''Rotates the given numpy matrix by 180 degree counterclockwise.'''\n",
    "        return np.rot90(np.rot90(mtx))\n",
    "\n",
    "    def rotate_thrice(mtx):\n",
    "        '''Rotates the given numpy matrix by 180 degree counterclockwise.'''\n",
    "        return np.rot90(np.rot90(np.rot90(mtx)))\n",
    "\n",
    "    def flip_horizontally(mtx):\n",
    "        '''Retruns a horizontal mirror of the given numpy matrix.'''\n",
    "        return mtx[::-1]\n",
    "\n",
    "    def flip_vertically(mtx):\n",
    "        '''Retruns a vertical mirror of the given numpy matrix.'''\n",
    "        shape = mtx.shape\n",
    "        output = np.zeros(shape)\n",
    "\n",
    "        for row, clm in np.ndindex(shape):\n",
    "            output[row][clm] = mtx[row][shape[1] - 1 - clm]\n",
    "\n",
    "        return output\n",
    "\n",
    "    def flip_diagonally(mtx):\n",
    "        '''Returns the transpose of the given numpy matrix along the major diagonal.'''\n",
    "        # D1 = M2 R1\n",
    "        output = flip_horizontally(rotate_once(mtx))\n",
    "        return output\n",
    "\n",
    "    def flip_antidiagonally(mtx):\n",
    "        '''Returns the transpose of the givne numpy matrix along the minor diagonal.'''\n",
    "        # D2 = R1 M2\n",
    "        output = rotate_once(flip_horizontally(mtx))\n",
    "        return output\n",
    "    \n",
    "#     # Checks if a folder with the same name exist; if so, delete it and then create one; if not create one.\n",
    "#     if os.path.exists(output_folder_dir):\n",
    "#         subprocess.call(['rm', '-r', output_folder_dir])\n",
    "#         subprocess.call(['mkdir', output_folder_dir])\n",
    "#     else:\n",
    "#         subprocess.call(['mkdir', output_folder_dir])\n",
    "    \n",
    "    image = get_fits_data(input_folder_dir + input_name, method='raw')\n",
    "    \n",
    "    new_image = rotate_none(image)                  # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_R0.fits', replace_duplicates=True)\n",
    "        \n",
    "    new_image = rotate_once(image)         # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_R1.fits', replace_duplicates=True)\n",
    "    \n",
    "    new_image = rotate_twice(image)        # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_R2.fits', replace_duplicates=True)\n",
    "    \n",
    "    new_image = rotate_thrice(image)       # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_R3.fits', replace_duplicates=True)\n",
    "    \n",
    "    new_image = flip_horizontally(image)   # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_MH.fits', replace_duplicates=True)\n",
    "    \n",
    "    new_image = flip_vertically(image)     # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_MV.fits', replace_duplicates=True)\n",
    "    \n",
    "    new_image = flip_diagonally(image)     # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_D1.fits', replace_duplicates=True)\n",
    "    \n",
    "    new_image = flip_antidiagonally(image) # Transform the image\n",
    "    save_fits(new_image, output_folder_dir + input_name[:-5] + '_D2.fits', replace_duplicates=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auto_set_maker(mother_folder, n_half_test):\n",
    "    '''Specify a mother_folder which contains two subfolders: raws and lenses. \n",
    "    It will create a training and testing dataset from them, \n",
    "    where the testing dataset would have 2 x n_half_test lines.'''\n",
    "    \n",
    "    mother_folder = './new_sets/'\n",
    "    lenses_dir = mother_folder + 'lenses/'\n",
    "    lenses_x8_dir = lenses_dir[:-1] + '_x8/'\n",
    "    %mkdir $lenses_x8_dir\n",
    "    \n",
    "    lens_names = list_file_names(lenses_dir)\n",
    "\n",
    "    for name in lens_names:\n",
    "        transform_fits(input_folder_dir=lenses_dir, input_name=name, output_folder_dir=lenses_x8_dir)\n",
    "\n",
    "    raws_dir = mother_folder + 'raws/'\n",
    "    raws_x8_dir = raws_dir[:-1] + '_x8/'\n",
    "    %mkdir $raws_x8_dir\n",
    "    \n",
    "    raw_names = list_file_names(raws_dir)\n",
    "\n",
    "    for name in lens_names:\n",
    "        transform_fits(input_folder_dir=raws_dir, input_name=name, output_folder_dir=raws_x8_dir)\n",
    "\n",
    "    raws_x8_csv_dir   = list_and_label_files(raws_x8_dir, file_label='unlensed')\n",
    "\n",
    "    lenses_x8_csv_dir = list_and_label_files(lenses_x8_dir, file_label='lensed')\n",
    "\n",
    "    create_test_and_train_csv_data_sets_v2(raws_csv_dir=raws_x8_csv_dir, \n",
    "                                           lenses_csv_dir=lenses_x8_csv_dir, \n",
    "                                           train_csv_dir=mother_folder + 'train_01.csv', \n",
    "                                           test_csv_dir=mother_folder  + 'test_01.csv',\n",
    "                                           n_half_test=n_half_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auto_transform(folder_dir):\n",
    "    output_dir = folder_dir[:-1] + '_x8/'\n",
    "    %mkdir $output_dir\n",
    "    \n",
    "    for name in list_file_names(folder_dir):\n",
    "        transform_fits(input_folder_dir=folder_dir, input_name=name, output_folder_dir=output_dir)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Bunch of function I ran:\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/good_lenses/', file_label='lensed')\n",
    "# names = list_file_names('./COSMOS_ASC_055/good_lenses/', shuffle_names=False)\n",
    "\n",
    "# for name in names:\n",
    "#     transform_fits('./COSMOS_ASC_055/good_lenses/', name, './COSMOS_ASC_055/good_lenses_x8/')\n",
    "    \n",
    "# list_and_label_files('./COSMOS_ASC_055/good_lenses_x8/', file_label='lensed')\n",
    "\n",
    "# ----\n",
    "\n",
    "# for i in range(1, 10):\n",
    "#     fd = './COSMOS_ASC_055/raws/raw_000'+ str(i) + '.fits'\n",
    "#     save_fits(enhance_fits_2(fd), save_to='/home/USER/Desktop/abc_' + str(i) + '.fits', replace_duplicates=True)\n",
    "\n",
    "# save_fits(enhance_fits_2('./COSMOS_ASC_055/raws/raw_3589.fits'), save_to='/home/USER/Desktop/not_from_TF.fits', replace_duplicates=True)\n",
    "\n",
    "# os.path.exists('/home/USER/Desktop/abc_1.fits')\n",
    "\n",
    "# subprocess.check_call('gvfs-trash '/home/USER/Desktop/abc_1.fits'')\n",
    "\n",
    "# merge_two_fits('/home/USER/Desktop/abc_2.fits', '/home/USER/Desktop/abc_3.fits', 3, '/home/USER/Desktop/cab.fits')\n",
    "\n",
    "# transform_fits('/home/USER/Desktop/','abc_1.fits', '/home/USER/Desktop/')\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/raws/', file_label='unlensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Bunch of function I ran:\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/good_lenses/', file_label='lensed')\n",
    "# names = list_file_names('./COSMOS_ASC_055/good_lenses/', shuffle_names=False)\n",
    "\n",
    "# for name in names:\n",
    "#     transform_fits('./COSMOS_ASC_055/good_lenses/', name, './COSMOS_ASC_055/good_lenses_x8/')\n",
    "    \n",
    "# list_and_label_files('./COSMOS_ASC_055/good_lenses_x8/', file_label='lensed')\n",
    "\n",
    "# ----\n",
    "\n",
    "# for i in range(1, 10):\n",
    "#     fd = './COSMOS_ASC_055/raws/raw_000'+ str(i) + '.fits'\n",
    "#     save_fits(enhance_fits_2(fd), save_to='/home/USER/Desktop/abc_' + str(i) + '.fits', replace_duplicates=True)\n",
    "\n",
    "# save_fits(enhance_fits_2('./COSMOS_ASC_055/raws/raw_3589.fits'), save_to='/home/USER/Desktop/not_from_TF.fits', replace_duplicates=True)\n",
    "\n",
    "# os.path.exists('/home/USER/Desktop/abc_1.fits')\n",
    "\n",
    "# subprocess.check_call('gvfs-trash '/home/USER/Desktop/abc_1.fits'')\n",
    "\n",
    "# merge_two_fits('/home/USER/Desktop/abc_2.fits', '/home/USER/Desktop/abc_3.fits', 3, '/home/USER/Desktop/cab.fits')\n",
    "\n",
    "# transform_fits('/home/USER/Desktop/','abc_1.fits', '/home/USER/Desktop/')\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/raws/', file_label='unlensed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Auxiliary Codes\n",
    "This sections contians the code I wrote to do small tasks that might be needed no longer. They are here just in case."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### I have shifted the followings to LensingFlow itself."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# def enhance_fits_1(file_directory):\n",
    "#     '''This function will go to a given directory, extract the fits image data.\n",
    "#     Then it sets negative pixels to zero, normalizes it, and transforms each pixel by log2(1e+5 * pixel + 1e-10).\n",
    "#     Read the comments for the remaining processes.'''\n",
    "    \n",
    "#     #and returns a normalized version of it.\n",
    "    \n",
    "#     image_data     = fits.getdata(file_directory)\n",
    "    \n",
    "#     # Shifting pixel values up so the lowest one is zero:\n",
    "#     image_data     = image_data + max(-np.min(image_data),0)\n",
    "    \n",
    "#     # Normalize\n",
    "#     image_data     = image_data/np.sum(image_data)    \n",
    "    \n",
    "#     # Multiply by a high factor:\n",
    "#     image_data = image_data*100000\n",
    "    \n",
    "#     # Take log of each pixel\n",
    "#     image_data = np.log2(image_data + 1e-10)\n",
    "    \n",
    "#     #shifting pixel values up so the lowest one is zero\n",
    "#     image_data     = image_data + max(-np.min(image_data),0)\n",
    "    \n",
    "#     median = np.median(image_data)\n",
    "    \n",
    "#     image_data = image_data - median\n",
    "#     image_data = np.maximum(image_data, 0)\n",
    "    \n",
    "#     #normalizing\n",
    "#     image_data     = image_data/np.sum(image_data)\n",
    "    \n",
    "#     # Let's try divding by the highest pixel value\n",
    "#     image_data     = image_data/np.max(image_data)\n",
    "    \n",
    "#     return image_data\n",
    "################################################################################################################\n",
    "# def enhance_fits_2(file_dir):\n",
    "#     '''\n",
    "#     1. Gets FITS data from the given file_dir.\n",
    "#     2. Evaluates mean pixel value.\n",
    "#     3. Subtracts the mean from all pixels.\n",
    "#     4. Divide by the largest pixel magnitude\n",
    "#     5. returns the updated image_data.'''\n",
    "    \n",
    "#     # 1.\n",
    "#     image_data = fits.getdata(file_dir)\n",
    "    \n",
    "#     # 2.\n",
    "#     mean       = np.mean(image_data)\n",
    "    \n",
    "#     # 3. \n",
    "#     image_data = image_data - mean\n",
    "    \n",
    "#     # 4.\n",
    "#     norm_coef = np.max(np.abs(image_data))\n",
    "#     if norm_coef != 0:\n",
    "#         image_data = image_data/np.max(np.abs(image_data))\n",
    "    \n",
    "#     # 5.\n",
    "#     return image_data\n",
    "################################################################################################################\n",
    "# def enhance_fits_3(file_dir):\n",
    "#     '''\n",
    "#     1. Gets FITS data from the given file_dir.\n",
    "#     2. Evaluates mean pixel value.\n",
    "#     3. Subtracts the mean from all pixels.\n",
    "#     4. Divide by the largest pixel magnitude\n",
    "#     5. returns the updated image_data.'''\n",
    "    \n",
    "#     # 1.\n",
    "#     image_data = fits.getdata(file_dir)\n",
    "    \n",
    "#     # 2.\n",
    "#     mean       = np.mean(image_data)\n",
    "    \n",
    "#     # 3. \n",
    "#     image_data = image_data - mean\n",
    "    \n",
    "#     # 4.\n",
    "#     norm_coef = np.std(image_data)\n",
    "\n",
    "#     if norm_coef != 0:\n",
    "#         image_data = image_data/norm_coef\n",
    "    \n",
    "#     # 5.\n",
    "#     return image_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "editable": true
   },
   "outputs": [],
   "source": [
    "# def create_enhanced_raws(src_folder_dir):\n",
    "#     '''Takes fits files and adjusts the contrast using \"enhance_fits(...)\" function.\n",
    "#     New fits files will be saved at folder_name_enhanced located in the upper directory of the folder \"folder_name\".\n",
    "#     Returns dst_folder_dir.'''\n",
    "    \n",
    "#     dst_folder_dir = src_folder_dir[:-1] + '_enhanced/'\n",
    "    \n",
    "#     # If dst_folder_dir doesn't exist, then create it:\n",
    "#     if not os.path.exists(dst_folder_dir): os.makedirs(dst_folder_dir)\n",
    "\n",
    "#     src_file_names = []\n",
    "\n",
    "#     # Add all file names in src_folder_dir to src_file_names:\n",
    "#     for (dir_path, dir_name, file_name) in os.walk(src_folder_dir):\n",
    "#         src_file_names.extend(file_name)\n",
    "\n",
    "#     # Normalize original FITS files and save them to dst_folder_dir:\n",
    "#     for file_name in src_file_names:\n",
    "#         new_image = enhance_fits(src_folder_dir + file_name)\n",
    "#         save_fits(data_array=new_image, save_to=dst_folder_dir + file_name, original_fits_dir=src_folder_dir + file_name, match=True)\n",
    "    \n",
    "#     return dst_folder_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for i in rng:\n",
    "#     print(folder_dirs[i])\n",
    "#     create_enhanced_raws(src_folder_dir=(folder_dirs[i] + 'raws/'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Creating a training set for auto enhancement which failed. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# folder1_dir  = '/home/USER/Desktop/folder1/'  # bad  lenses\n",
    "# folder2_dir  = '/home/USER/Desktop/folder2/'  # good lenses\n",
    "# folder11_dir = '/home/USER/Desktop/folder11/' # bad  lenses x 800\n",
    "# folder22_dir = '/home/USER/Desktop/folder22/' # good lenses x 800\n",
    "\n",
    "# l1 = list_file_names(folder1_dir, shuffle_names=False)\n",
    "# l2 = list_file_names(folder2_dir, shuffle_names=False)\n",
    "\n",
    "# for name in l1:\n",
    "#     transform_fits(folder1_dir, name, folder11_dir)\n",
    "\n",
    "# for name in l2:\n",
    "#     transform_fits(folder2_dir, name, folder22_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# create_enhanced_raws('COSMOS_ASC_055/776_bad_lenses/')\n",
    "# create_enhanced_raws('COSMOS_ASC_055/824_good_lenses/')\n",
    "# list_and_label_files('COSMOS_ASC_055/776_bad_lenses_enhanced/', 'unlensed')\n",
    "# list_and_label_files('COSMOS_ASC_055/824_good_lenses_enhanced/', 'lensed')  \n",
    "\n",
    "# create_test_and_train_csv_data_sets(raws_csv_dir='COSMOS_ASC_055/776_bad_lenses_enhanced.csv', \n",
    "#                                     lenses_csv_dir='COSMOS_ASC_055/824_good_lenses_enhanced.csv', \n",
    "#                                     train_csv_dir='COSMOS_ASC_055/enhanced_good_lens_train.csv', \n",
    "#                                     test_csv_dir='COSMOS_ASC_055/enhanced_good_lens_test.csv', \n",
    "#                                     n_raws_train=700, \n",
    "#                                     n_lenses_train=700)\n",
    "\n",
    "# create_enhanced_raws('COSMOS_ASC_055/lenses/')\n",
    "# list_and_label_files('COSMOS_ASC_055/lenses_enhanced/', 'good_or_bad_sim_lens')\n",
    "# create_enhanced_raws('COSMOS_ASC_055/raws/')\n",
    "# list_and_label_files('COSMOS_ASC_055/raws_enhanced/', 'unlensed')\n",
    "# list_and_label_files('COSMOS_ASC_055/lenses_enhanced/', 'lensed')\n",
    "\n",
    "# create_test_and_train_csv_data_sets(raws_csv_dir='COSMOS_ASC_055/raws_enhanced.csv', \n",
    "#                                     lenses_csv_dir='COSMOS_ASC_055/lenses_enhanced.csv', \n",
    "#                                     train_csv_dir='COSMOS_ASC_055/train_01.csv', \n",
    "#                                     test_csv_dir='COSMOS_ASC_055/test_01.csv', \n",
    "#                                     n_raws_train=3816-500, \n",
    "#                                     n_lenses_train=3816-500)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plotting unfiltered learning accuracy vs number of itterations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Plotting unfiltered learning accuracy vs number of itterations\n",
    "# data = np.genfromtxt('./Extras/training_stats_wit_out_filteration.txt', \n",
    "#                      delimiter=',', \n",
    "#                      names=['x', 'y'])\n",
    "\n",
    "# opt_fig = plt.figure()\n",
    "# ax1 = opt_fig.add_subplot(1, 1, 1)\n",
    "# ax1.plot(data['x'], data['y'], 'r-')\n",
    "# ax1.set_xlabel('Number of Iterations')\n",
    "# ax1.set_ylabel('Accuracy (%)')\n",
    "# ax1.set_ylim([50, 100])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plotting raw vs enhanced FITS examples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# raw1 = get_fits_data('/home/USER/Desktop/4_lenses/1.fits', method='divide by max')\n",
    "# raw2 = get_fits_data('/home/USER/Desktop/4_lenses/2.fits', method='divide by max')\n",
    "# raw3 = get_fits_data('/home/USER/Desktop/4_lenses/3.fits', method='divide by max')\n",
    "# raw4 = get_fits_data('/home/USER/Desktop/4_lenses/4.fits', method='divide by max')\n",
    "\n",
    "# enh1 = get_fits_data('/home/USER/Desktop/4_lenses_enhanced/1.fits')\n",
    "# enh2 = get_fits_data('/home/USER/Desktop/4_lenses_enhanced/2.fits')\n",
    "# enh3 = get_fits_data('/home/USER/Desktop/4_lenses_enhanced/3.fits')\n",
    "# enh4 = get_fits_data('/home/USER/Desktop/4_lenses_enhanced/4.fits')\n",
    "\n",
    "# raw_enh_0 = [(raw1, enh1),(raw2, enh2),(raw3, enh3),(raw4, enh4)]\n",
    "# raw_enh_1 = [raw1, enh1, raw2, enh2, raw3, enh3, raw4, enh4]\n",
    "# raw_enh_2 = [raw1, raw2, raw3, raw4, enh1, enh2, enh3, enh4]\n",
    "# labels    = ['(a.1.)', '(a.2.)', '(a.3.)', '(a.4.)', '(b.1.)', '(b.2.)', '(b.3.)', '(b.4.)']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# a_min = np.min(raw_enh_2[:4])\n",
    "# a_max = np.max(raw_enh_2[:4])\n",
    "\n",
    "# b_min = np.min(raw_enh_2[4:])\n",
    "# b_max = np.max(raw_enh_2[4:])\n",
    "\n",
    "# a_min = b_min = -0.1\n",
    "# a_max = b_max = +1\n",
    "\n",
    "# print(str(a_min) +', '+ str(a_max) +', '+ str(b_min) +', '+ str(b_max))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# fig, axes = plt.subplots(2, 4)\n",
    "\n",
    "# for i, ax in enumerate(axes.flat[:4]):\n",
    "#     ax.imshow(raw_enh_2[i], vmin=a_min, vmax=a_max)\n",
    "#     ax.set_xticks([])\n",
    "#     ax.set_yticks([])\n",
    "#     ax.set_xlabel(labels[i])\n",
    "\n",
    "# for i, ax in enumerate(axes.flat[4:]):\n",
    "#     im = ax.imshow(raw_enh_2[4+i], vmin=b_min, vmax=b_max)\n",
    "#     ax.set_xticks([])\n",
    "#     ax.set_yticks([])\n",
    "#     ax.set_xlabel(labels[4+i])\n",
    "# fig.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Bunch of function I ran:\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/good_lenses/', file_label='lensed')\n",
    "# names = list_file_names('./COSMOS_ASC_055/good_lenses/', shuffle_names=False)\n",
    "\n",
    "# for name in names:\n",
    "#     transform_fits('./COSMOS_ASC_055/good_lenses/', name, './COSMOS_ASC_055/good_lenses_x8/')\n",
    "    \n",
    "# list_and_label_files('./COSMOS_ASC_055/good_lenses_x8/', file_label='lensed')\n",
    "\n",
    "# ----\n",
    "\n",
    "# for i in range(1, 10):\n",
    "#     fd = './COSMOS_ASC_055/raws/raw_000'+ str(i) + '.fits'\n",
    "#     save_fits(enhance_fits_2(fd), save_to='/home/USER/Desktop/abc_' + str(i) + '.fits', replace_duplicates=True)\n",
    "\n",
    "# save_fits(enhance_fits_2('./COSMOS_ASC_055/raws/raw_3589.fits'), save_to='/home/USER/Desktop/not_from_TF.fits', replace_duplicates=True)\n",
    "\n",
    "# os.path.exists('/home/USER/Desktop/abc_1.fits')\n",
    "\n",
    "# subprocess.check_call('gvfs-trash '/home/USER/Desktop/abc_1.fits'')\n",
    "\n",
    "# merge_two_fits('/home/USER/Desktop/abc_2.fits', '/home/USER/Desktop/abc_3.fits', 3, '/home/USER/Desktop/cab.fits')\n",
    "\n",
    "# transform_fits('/home/USER/Desktop/','abc_1.fits', '/home/USER/Desktop/')\n",
    "\n",
    "# list_and_label_files(folder_dir='./COSMOS_ASC_055/raws/', file_label='unlensed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### This block of code will copy the lenses I have examined to a separate folder."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # This block of code will copy the lenses I have examined to a separate folder.\n",
    "# l_milad   = [3, 6, 9, 13, 16, 19, 20, 22, 25, 26, 29, 31, 33, 35, 36, 37, 42, 43, 44, 45, 48, 52, 54, 55, 57, 59, 61, 62, 65, 66, 67, 68, 69, 71, 73, 75, 76, 77, 78, 80, 83, 84, 88, 89, 90, 93, 100, 101, 104, 107, 108, 109, 110, 112, 113, 115, 118, 119, 121, 131, 132, 139, 140, 142, 145, 147, 148, 149, 151, 152, 153, 154, 156, 157, 158, 159, 160, 163, 164, 165, 166, 175, 178, 180, 181, 182, 185, 186, 189, 192, 193, 194, 195, 196, 201, 202, 203, 204, 218, 219, 221, 223, 224, 226, 226, 229, 230, 231, 232, 238, 240, 246, 247, 248, 250, 252, 255, 262, 264, 265, 270, 280, 282, 284, 285, 286, 287, 288, 289, 291, 297, 298, 299, 301, 304, 307, 311, 312, 313, 314, 317, 331, 338, 343, 345, 346, 347, 353, 355, 356, 360, 373, 379, 380, 388, 390, 398, 400, 401, 402, 403, 405, 406, 407, 408, 411, 413, 414, 415, 416, 418, 419, 420, 422, 424, 425, 426, 430, 433, 434, 436, 437, 438, 439, 440, 441, 442, 446, 451, 456, 457, 458, 459, 461, 462, 463, 464, 465, 470, 471, 472, 473, 474, 479, 481, 483, 486, 489, 490, 491, 492, 498, 499, 500, 505, 506, 508, 510, 514, 519, 520, 524, 526, 528, 532, 545, 549, 550, 557, 578, 593, 598, 604, 605, 606, 607, 620, 624, 629, 635, 636, 642, 645, 647, 648, 657, 658, 664, 665, 667, 669, 672, 675, 676, 680, 683, 686, 688, 692, 695, 700, 710, 711, 712, 714, 715, 719, 725, 731, 733, 734, 736, 740, 741, 742, 743, 744, 745, 748, 751, 752, 753, 767, 768, 781, 782, 784, 786, 788, 789, 791, 793, 797, 798, 801, 811, 812, 818, 824, 828, 831, 835, 840, 842, 844, 846, 847, 859, 862, 863, 867, 876, 882, 883, 885, 886, 889, 893, 894, 896, 899, 902, 903, 905, 906, 907, 908, 910, 911, 914, 915, 917, 919, 920, 921, 926, 928, 930, 932, 941, 945, 951, 954, 955, 961, 962, 963, 964, 966, 968, 970, 975, 976, 977, 978, 980, 981, 982, 983, 984, 985, 987, 993, 997]\n",
    "# l_aroosa  = [2000, 2001, 2004, 2005, 2009, 2015, 2017, 2018, 2022, 2025, 2028, 2030, 2077, 2031, 2033, 2042, 2046, 2051, 2053, 2062, 2063, 2066, 2069, 2071, 2072, 2074, 2075, 2077, 2078, 2081, 2082, 2088, 2091, 2092, 2093, 2096, 2098, 2102, 2106, 2107, 2110, 2111, 2113, 2114, 2118, 2122, 2123, 2124, 2129, 2132, 2133, 2134, 2135, 2136, 2144, 2145, 2147, 2148, 2149, 2152, 2154, 2156, 2157, 2158, 2160, 2164, 2166, 2166, 2167, 2174, 2177, 2180, 2182, 2185, 2187, 2188, 2198, 2200, 2207, 2209, 2210, 2211, 2214, 2222, 2224, 2225, 2228, 2231, 2232, 2233, 2238, 2239, 2240, 2241, 2247, 2248, 2257, 2258, 2261, 2266, 2268, 2261, 2271, 2272, 2275, 2275, 2279, 2284, 2291, 2294, 2297, 2298, 2230, 2308, 2309, 2312, 2315, 2318, 2319, 2325, 2332, 2336, 2338, 2344, 2349, 2350, 2356, 2361, 2362, 2363, 2365, 2368, 2369, 2370, 2371, 2373, 2375, 2379, 2380, 2395, 2398, 2400, 2401, 2403, 2409, 2410, 2411, 2414, 2415, 2419, 2420, 2421, 2422, 2423, 2425, 2427, 2428, 2429, 2431, 2434, 2435, 2437, 2438, 2440, 2441, 2442, 2447, 2449, 2450, 2452, 2455, 2458, 2459, 2460, 2462, 2464, 2466, 2469, 2471, 2472, 2474, 2476, 2479, 2480, 2482, 2485, 2489, 2491, 2497, 2502, 2503, 2504, 2505, 2512, 2516, 2520, 2521, 2525, 2527, 2528, 2530, 2534, 2535, 2538, 2540, 2541, 2543, 2544, 2545, 2548, 2549, 2550, 2552, 2553, 2554, 2564, 2563, 2570, 2574, 2578, 2582, 2584, 2588, 2589, 2593, 2595, 2592, 2600, 2601, 2604, 2606, 2608, 2610, 2613, 2616, 2620, 2621, 2622, 2624, 2627, 2641, 2642, 2643, 2644, 2643, 2649, 2651, 2675, 2660, 2661, 2664, 2668, 2670, 2673, 2696, 2699, 2700, 2701, 2703, 2705, 2718, 2701, 2703, 2705, 2706, 2707, 2716, 2717, 2718, 2722, 2723, 2725, 2726, 2727, 2728, 2733, 2735, 2736, 2740, 2741, 2745, 2746, 2749, 2750, 2753, 2756, 2757, 2758, 2760, 2761, 2765, 2772, 2774, 2775, 2778, 2779, 2792, 2795, 2800, 2801, 2803, 2809, 2813, 2814, 2815, 2820, 2823, 2831, 2832, 2033, 2834, 2839, 2843, 2851, 2852, 2853, 2854, 2855, 2856, 2857, 2858, 2864, 2866, 2867, 2869, 2870, 2871, 2875, 2879, 2880, 2881, 2893, 2895, 2897, 2898, 2899, 2900, 2903, 2907, 2909, 2910, 2913, 2918, 2935, 2938, 2939, 2940, 2941, 2943, 2944, 2946, 2948, 2950, 2951, 2956, 2958, 2961, 2962, 2969, 2970, 2971, 2974, 2981, 2983, 2985, 2990, 2991, 2994, 2995, 2999] # Aroosa took after 2198\n",
    "# l_noah    = [3001, 3010, 3011, 3012, 3014, 3015, 3016, 3018, 3019, 3020, 3022, 3025, 3028, 3033, 3034, 3043, 3045, 3048, 3050, 3051, 3057, 3059, 3065, 3072, 3076, 3078, 3082, 3086, 3089, 3094, 3096, 3097, 3098, 3106, 3107, 3111, 3112, 3117, 3119, 3120, 3121, 3125, 3127, 3132, 3137, 3138, 3140, 3143, 3144, 3145, 3146, 3152, 3153, 3157, 3158, 3160, 3163, 3164, 3167, 3168, 3170, 3171, 3173, 3179, 3180, 3183, 3184, 3185, 3187, 3189, 3196, 3199, 3200, 3203, 3205, 3206, 3211, 3213, 3214, 3216, 3221, 3230, 3231, 3233, 3241, 3243, 3248, 3249, 3725, 3253, 3257, 3259, 3260, 3262, 3263, 3267, 3269, 3271, 3276, 3281, 3285, 3286, 3287, 3288, 3289, 3292, 3294, 3295, 3296, 3297, 3299, 3300, 3303, 3310, 3313, 3325, 3333, 3335, 3340, 3341, 3342, 3345, 3346, 3348, 3351, 3352, 3354, 3363, 3368, 3374, 3380, 3384, 3386, 3390, 3392, 3396, 3397, 3399, 3401, 3405, 3408, 3418, 3420, 3425, 3428, 3432, 3434, 3445, 3460, 3463, 3472, 3474, 3475, 3479, 3485, 3486, 3490, 3492, 3498, 3504, 3507, 3510, 3511, 3512, 3513, 3514, 3515, 3523, 3527, 3530, 3533, 3536, 3539, 3548, 3551, 3567, 3568, 3572, 3574, 3581, 3583, 3589, 3590, 3599, 3602, 3603, 3607, 3611, 3612, 3627, 3628, 3629, 3634, 3638, 3640, 3642, 3643, 3646, 3648, 3649, 3651, 3655, 3656, 3682, 3693, 3694, 3695, 3700, 3702, 3703, 3704, 3707, 3708, 3710, 3711, 3714, 3718, 3727, 3735, 3737, 3740, 3745, 3749, 3752, 3753, 3754, 3758, 3760, 3767, 3769, 3772, 3776, 3777, 3778, 3780, 3781, 3782, 3787, 3790, 3796, 3802, 3813, 3814]\n",
    "# l_special = [341, 396, 494, 529, 579, 612, 706, 758, 799, 807, 854, 881, 897, 901, 904, 986, 996, 2117] # Very good lenses, to be tranformed for training\n",
    "# l_all     = l_milad + l_noah + l_aroosa + l_special\n",
    "\n",
    "# def copy_lens(index,\n",
    "#         input_folder_dir  = './COSMOS_ASC_055/lenses/',\n",
    "#         output_folder_dir = './COSMOS_ASC_055/good_lenses/'):\n",
    "        \n",
    "#     src_dir = input_folder_dir  + 'lens_' + str(index) + '.fits'\n",
    "#     dst_dir = output_folder_dir + 'lens_' + str(index).zfill(4) + '.fits'\n",
    "#     #!cp -u $src_dir $dst_dir\n",
    "#     subprocess.call('cp -u ' + src_dir + ' ' + dst_dir, shell=True)\n",
    "    \n",
    "# # for index in (l_0_500):\n",
    "# #     copy_lens(index)\n",
    "\n",
    "# # for index in (l_2000_3000):\n",
    "# #     copy_lens(index)\n",
    "    \n",
    "# # for index in l_special:\n",
    "# #     copy_lens(index, output_folder_dir = './COSMOS_ASC_055/good_lenses_special/')\n",
    "# %mkdir './COSMOS_ASC_055/good_lenses/'\n",
    "# for index in l_all:\n",
    "#     copy_lens(index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# View multiple fits from the lists above\n",
    "def ds9_by_list(x):\n",
    "    selected_dirs = []\n",
    "\n",
    "    for i in x:\n",
    "        selected_dirs.append('./COSMOS_ASC_055/lenses/lens_' + str(i) + '.fits')\n",
    "    print(selected_dirs)\n",
    "    subprocess.check_call(\n",
    "        ['cd \"$1\" || exit; shift; exec ds9 \"$@\"', \"_\", ''] + selected_dirs,\n",
    "        shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def create_test_and_train_folders_from_above_lists():\n",
    "    random.shuffle(l_all)\n",
    "\n",
    "    n_test = 200\n",
    "    n_spcl = len(l_special)//2\n",
    "    n_rmdr = n_test - n_spcl\n",
    "\n",
    "    # Create testing set\n",
    "\n",
    "    # Add usual lenses\n",
    "    %mkdir './COSMOS_ASC_055/test_lenses/'\n",
    "    for index in l_all[:n_rmdr]:\n",
    "        copy_lens(index, output_folder_dir='./COSMOS_ASC_055/test_lenses/')\n",
    "\n",
    "    # Add special lenses\n",
    "    for index in l_special[:n_spcl]:\n",
    "        copy_lens(index, output_folder_dir='./COSMOS_ASC_055/test_lenses/')\n",
    "\n",
    "    # Create training set\n",
    "\n",
    "    # Add usual lenses\n",
    "    lens_train_dir = './COSMOS_ASC_055/train_lenses/'\n",
    "    %mkdir $lens_train_dir\n",
    "    for index in l_all[n_rmdr:]:\n",
    "        copy_lens(index, output_folder_dir=lens_train_dir)\n",
    "\n",
    "    # Add special lenses\n",
    "    for index in l_special[n_spcl:]:\n",
    "        copy_lens(index, output_folder_dir=lens_train_dir)\n",
    "\n",
    "    # Transform training set\n",
    "    %mkdir './COSMOS_ASC_055/train_lenses_x8/'\n",
    "    train_lens_names = list_file_names(lens_train_dir, shuffle_names=False)\n",
    "    for name in train_lens_names:\n",
    "        transform_fits(lens_train_dir, name, './COSMOS_ASC_055/train_lenses_x8/')\n",
    "        \n",
    "    list_and_label_files('./COSMOS_ASC_055/train_lenses_x8/', 'lensed')\n",
    "    \n",
    "    list_and_label_files('./COSMOS_ASC_055/test_lenses/', 'lensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def create_test_and_train_datasets_from_above_lists():\n",
    "    names = list_file_names('./COSMOS_ASC_055/raws/')\n",
    "    %mkdir './COSMOS_ASC_055/test_raws/'\n",
    "    for name in names[:200]:\n",
    "        file_dir = './COSMOS_ASC_055/raws/' + name\n",
    "        !mv $file_dir './COSMOS_ASC_055/test_raws/'\n",
    "\n",
    "    list_and_label_files('./COSMOS_ASC_055/test_raws/', file_label='unlensed')\n",
    "    list_and_label_files('./COSMOS_ASC_055/raws/', file_label='unlensed')\n",
    "\n",
    "    merge_two_csv_files(csv_dir_1='./COSMOS_ASC_055/test_lenses.csv',\n",
    "                        csv_dir_2='./COSMOS_ASC_055/test_raws.csv', \n",
    "                        output_csv_dir='./COSMOS_ASC_055/test_dataset.csv',\n",
    "                        shuffle_lines=True)\n",
    "\n",
    "    merge_two_csv_files(csv_dir_1='./COSMOS_ASC_055/train_lenses_x8.csv',\n",
    "                        csv_dir_2='./COSMOS_ASC_055/raws.csv',\n",
    "                        output_csv_dir='./COSMOS_ASC_055/train_dataset.csv',\n",
    "                        shuffle_lines=True,\n",
    "                        match_len=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# enhance_fits_2('/home/USER/Desktop/raw_0004.fits')\n",
    "\n",
    "# enhance_fits_2('/home/USER/Desktop/raw_0005.fits')\n",
    "\n",
    "# enhance_fits_2('/home/USER/Desktop/raw_0006.fits')\n",
    "\n",
    "# multi_ds9('./COSMOS_ASC_055/LF_01Mar2017_Results_01.csv', range(0, 100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#list_and_label_files('./COSMOS_ASC_055/raws/', file_label='unlensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# import sys\n",
    "# import time\n",
    "# for i in range(10):\n",
    "#     sys.stdout.write(\"\\r{0}>\".format(\"=\"*i))\n",
    "#     sys.stdout.flush()\n",
    "#     time.sleep(0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# x = list_file_names('./Extras/lens_a2r/', shuffle_names=False)\n",
    "\n",
    "# %mkdir './Extras/lens_a2r_x8/'\n",
    "# for name in x:\n",
    "#     transform_fits('./Extras/lens_a2r/', name, './Extras/lens_a2r_x8/')\n",
    "\n",
    "# list_and_label_files('./Extras/lens_a2r_x8/', file_label='lensed')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "At this point, I have decided to select 100 adequate sources and add 100 adequate lenses to them. Then transform them separately and create a training set of 700+700 and a testing set of 100+100 from them.  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # good_sources = [3, 4, 5, 9, 11, 13, 14, 28, 29, 40, 42, 30, 62, 72, 73, 74, 75, 77, 80, 86, 89, 102, 104, 105, 260, 262, 280, 296, 348, 342, 432, 460, 514, 516, 550, 600, 609, 611, 639, 705, 682, 709, 757, 761, 769, 774, 776, 784, 785, 813, 815, 836, 867, 894, 903, 909, 910, 912, 913, 916, 926, 948, 971, 958, 961, 964, 966, 970, 973, 974, 977, 978, 1005, 1006, 1008, 1015, 1016, 1399, 1401, 1423, 1472, 1473, 1536, 1555, 1559, 1560, 1566, 1568, 1579, 1582, 1589, 1594, 1599, 1600, 1605, 1614, 1615, 1617, 1620, 1830]\n",
    "\n",
    "# # i = 0\n",
    "\n",
    "# def copy_raws(index,\n",
    "#         input_folder_dir  = './COSMOS_ASC_055/lenses/',\n",
    "#         output_folder_dir = './COSMOS_ASC_055/good_lenses/'):\n",
    "#     global i\n",
    "#     src_dir = input_folder_dir  + 'raw_' + str(index).zfill(4) + '.fits'\n",
    "#     dst_dir = output_folder_dir + 'raw_' + str(i).zfill(2) + '.fits'\n",
    "#     #!cp -u $src_dir $dst_dir\n",
    "#     subprocess.call('cp -u ' + src_dir + ' ' + dst_dir, shell=True)\n",
    "    \n",
    "# # for j in good_sources:\n",
    "# #     copy_raws(index=j, input_folder_dir='./COSMOS_ASC_055/raws/', output_folder_dir='./COSMOS_ASC_055/good_raws/')\n",
    "# #     i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# auto_merge(raws_folder_dir='./COSMOS_ASC_055/good_raws/', \n",
    "#            ring_folder_dir='./lenstool_kitchen_02/rings/', \n",
    "#            output_folder_dir='/home/USER/Desktop/abc2/', \n",
    "#            coef_range=(0.2, 0.6))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # gl1 = [5, 8, 10, 12, 15, 18, 20, 23, 27, 29, 30, 36, 37, 50, 52, 54, 55, 56, 59, 61, 80, 91, 92, 99]\n",
    "# # gl2 = [1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, 16, 20, 22, 25, 26, 27, 28, 29, 30, 33, 35, 36, 39, 41, 44, 45, 47, 48, 49, 50, 54, 55, 56, 57, 58, 60, 62, 63, 65, 68, 70, 71, 72, 73, 74, 75, 76, 78, 81, 82, 85, 89, 91, 95, 96, 99]\n",
    "# # gl3 = [186, 166, 219, 232, 345, 456, 496, 893, 889, 981, 2004, 2071, 2096, 2123, 2072, 2185, 791, 731, 528, 317]\n",
    "# # i = 83\n",
    "\n",
    "# def copy_lens(index,\n",
    "#         input_folder_dir,\n",
    "#         output_folder_dir):\n",
    "#     global i\n",
    "#     src_dir = input_folder_dir  + 'lens_' + str(index).zfill(4) + '.fits'\n",
    "#     dst_dir = output_folder_dir + 'lens_' + str(i).zfill(4) + '.fits'\n",
    "#     #!cp -u $src_dir $dst_dir\n",
    "#     subprocess.call('cp -u ' + src_dir + ' ' + dst_dir, shell=True)\n",
    "#     i += 1\n",
    "    \n",
    "# # for index in gl3:\n",
    "# #     #copy_lens(index, input_folder_dir='/home/USER/Desktop/abc2/', output_folder_dir='/home/USER/Desktop/def/')\n",
    "# #     copy_lens(index, input_folder_dir='./COSMOS_ASC_055/good_lenses/', output_folder_dir='/home/USER/Desktop/def/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# x=['a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r']\n",
    "# for char in x:\n",
    "#     single_crop('/home/USER/Desktop/cutouts_500/lens_'+ char +'.fits', xCoord=250, yCoord=250, cutout_shape=(200, 200), output_fits_dir='/home/USER/Desktop/lens_'+ char +'.fits')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def auto_set_maker(mother_folder, n_half_test):\n",
    "    '''Specify a mother_folder which contains two subfolders: raws and lenses. \n",
    "    It will create a training and testing dataset from them, \n",
    "    where the testing dataset would have 2 x n_half_test lines.'''\n",
    "    \n",
    "    lenses_dir = mother_folder + 'good_lenses/' #<<< adjust\n",
    "    lenses_x8_dir = lenses_dir[:-1] + '_x8/'\n",
    "    %mkdir $lenses_x8_dir\n",
    "    \n",
    "    lens_names = list_file_names(lenses_dir)\n",
    "\n",
    "    for name in lens_names:\n",
    "        transform_fits(input_folder_dir=lenses_dir, input_name=name, output_folder_dir=lenses_x8_dir)\n",
    "\n",
    "    raws_dir = mother_folder + 'raws/'\n",
    "    raws_x8_dir = raws_dir[:-1] + '_x8/'\n",
    "    %mkdir $raws_x8_dir\n",
    "    \n",
    "#     raw_names = list_file_names(raws_dir)\n",
    "\n",
    "#     for name in raw_names:\n",
    "#         transform_fits(input_folder_dir=raws_dir, input_name=name, output_folder_dir=raws_x8_dir)\n",
    "\n",
    "    raws_csv_dir   = list_and_label_files(raws_dir, file_label='unlensed')\n",
    "\n",
    "    lenses_x8_csv_dir = list_and_label_files(lenses_x8_dir, file_label='lensed')\n",
    "\n",
    "    create_test_and_train_csv_data_sets_v2(raws_csv_dir=raws_csv_dir, \n",
    "                                           lenses_csv_dir=lenses_x8_csv_dir, \n",
    "                                           train_csv_dir=mother_folder + 'train_01.csv', \n",
    "                                           test_csv_dir=mother_folder  + 'test_01.csv',\n",
    "                                           n_half_test=n_half_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Rename some lenses\n",
    "# src_dir = '/home/USER/Desktop/LensingFlow_02Mar2017/new_sets/lenses/'\n",
    "# dst_dir = '/home/USER/Desktop/lenses/'\n",
    "# names = list_file_names(src_dir)\n",
    "\n",
    "# for name in names:\n",
    "#     x = (name.replace('00', '40'))\n",
    "#     subprocess.check_call('cp -u ' + src_dir + name + ' ' + dst_dir + x, shell=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Create a better train and test set, I've also done some parts manually. I'll explain in the paper what I've done.\n",
    "\n",
    "# lens_dir = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/good_lenses/'\n",
    "# raw_dir = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/raws/'\n",
    "# lens_test_dir = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/lens_test/'\n",
    "# raw_test_dir = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/raw_test/'\n",
    "# test_csv_dir = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/test_01.csv'\n",
    "\n",
    "# lens_names = list_file_names(lens_dir, shuffle_names=True)\n",
    "# raw_names  = list_file_names(raw_dir,  shuffle_names=True)\n",
    "\n",
    "# %mkdir $lens_test_dir\n",
    "# %mkdir $raw_test_dir\n",
    "\n",
    "# for i in range(2**7):\n",
    "#     subprocess.check_call(['mv', lens_dir + lens_names[i], lens_test_dir + lens_names[i]])\n",
    "#     subprocess.check_call(['mv', raw_dir  + raw_names[i],  raw_test_dir  + raw_names[i]])\n",
    "    \n",
    "\n",
    "\n",
    "# raws_csv_dir   = list_and_label_files(raw_dir, file_label='unlensed')\n",
    "\n",
    "# lenses_x8_csv_dir = list_and_label_files(lens_dir[:-1]+'_x8/', file_label='lensed')\n",
    "# mother_folder ='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/'\n",
    "# create_test_and_train_csv_data_sets_v2(raws_csv_dir=raws_csv_dir, \n",
    "#                                        lenses_csv_dir=lenses_x8_csv_dir, \n",
    "#                                        train_csv_dir=mother_folder + 'train_01.csv', \n",
    "#                                        test_csv_dir=mother_folder  + 'test_01.csv',\n",
    "#                                        n_half_test=0)\n",
    "\n",
    "# auto_set_maker(mother_folder='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/', n_half_test=0)\n",
    "\n",
    "# a = list_and_label_files(lens_test_dir, file_label='lensed')\n",
    "# b = list_and_label_files(raw_test_dir, file_label='unlensed')\n",
    "# merge_two_csv_files(a, b, output_csv_dir=test_csv_dir, shuffle_lines=True)\n",
    "\n",
    "# list_and_label_files('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/COSMOS_test/', file_label='unknown')\n",
    "\n",
    "# # Even better:\n",
    "\n",
    "# raw_names = list_file_names(raw_dir)\n",
    "# raws_x8_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/raws_x8/'\n",
    "# for name in raw_names:\n",
    "#     transform_fits(input_folder_dir=raw_dir, input_name=name, output_folder_dir=raws_x8_dir)\n",
    "\n",
    "# list_and_label_files(raws_x8_dir, file_label='unlensed')\n",
    "\n",
    "# merge_two_csv_files('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/raws_x8.csv', '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/good_lenses_x8.csv', output_csv_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/train_02.csv', match_len=True, shuffle_lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# def generate_sex_files(dst_dir='/home/USER/Desktop/',\n",
    "#                        default_SExtractor_files_dir='/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/SE_Files/'):\n",
    "#     '''Copies default .conv, .nnw, and .psf from default_SExtractor_files_dir to dst_dir and\n",
    "#     writes .sex and .param files specified here.'''\n",
    "    \n",
    "#     sex_content = '''# Non-default configuration file for SExtractor 2.5.0\n",
    "#     # EB 2006-07-14\n",
    "#     #\n",
    "\n",
    "#     #-------------------------------- Catalog ------------------------------------\n",
    "\n",
    "#     CATALOG_NAME     cat.txt        # name of the output catalog\n",
    "#     CATALOG_TYPE     ASCII_HEAD     # NONE,ASCII,ASCII_HEAD, ASCII_SKYCAT,\n",
    "#                                     # ASCII_VOTABLE, FITS_1.0 or FITS_LDAC\n",
    "#     PARAMETERS_NAME  LF_01.param  # name of the file containing catalog contents\n",
    "\n",
    "#     #------------------------------- Extraction ----------------------------------\n",
    "\n",
    "#     DETECT_TYPE      CCD            # CCD (linear) or PHOTO (with gamma correction)\n",
    "#     DETECT_MINAREA   200            # minimum number of pixels above threshold\n",
    "#     DETECT_THRESH    1.5            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2\n",
    "#     ANALYSIS_THRESH  2            # <sigmas> or <threshold>,<ZP> in mag.arcsec-2\n",
    "\n",
    "#     FILTER           Y              # apply filter for detection (Y or N)?\n",
    "#     FILTER_NAME      default.conv   # name of the file containing the filter\n",
    "\n",
    "#     DEBLEND_NTHRESH  32             # Number of deblending sub-thresholds\n",
    "#     DEBLEND_MINCONT  0.005          # Minimum contrast parameter for deblending\n",
    "\n",
    "#     CLEAN            Y              # Clean spurious detections? (Y or N)?\n",
    "#     CLEAN_PARAM      1.0            # Cleaning efficiency\n",
    "\n",
    "#     MASK_TYPE        CORRECT        # type of detection MASKing: can be one of\n",
    "#                                     # NONE, BLANK or CORRECT\n",
    "\n",
    "#     WEIGHT_TYPE      NONE\n",
    "\n",
    "#     #------------------------------ Photometry -----------------------------------\n",
    "\n",
    "#     #PHOT_APERTURES   5              # MAG_APER aperture diameter(s) in pixels\n",
    "#     #PHOT_AUTOPARAMS  2.5, 3.5       # MAG_AUTO parameters: <Kron_fact>,<min_radius>\n",
    "#     #PHOT_PETROPARAMS 2.0, 3.5       # MAG_PETRO parameters: <Petrosian_fact>,\n",
    "#                                     # <min_radius>\n",
    "\n",
    "#     #SATUR_LEVEL      50000.0        # level (in ADUs) at which arises saturation\n",
    "\n",
    "#     #MAG_ZEROPOINT    0.0            # magnitude zero-point\n",
    "#     #MAG_GAMMA        4.0            # gamma of emulsion (for photographic scans)\n",
    "#     #GAIN             0.0            # detector gain in e-/ADU\n",
    "#     #PIXEL_SCALE      1.0            # size of pixel in arcsec (0=use FITS WCS info)\n",
    "\n",
    "#     #------------------------- Star/Galaxy Separation ----------------------------\n",
    "\n",
    "#     SEEING_FWHM      1.2            # stellar FWHM in arcsec\n",
    "#     STARNNW_NAME     default.nnw    # Neural-Network_Weight table filename\n",
    "\n",
    "#     #------------------------------ Background -----------------------------------\n",
    "\n",
    "#     BACK_SIZE        64             # Background mesh: <size> or <width>,<height>\n",
    "#     BACK_FILTERSIZE  3              # Background filter: <size> or <width>,<height>\n",
    "\n",
    "#     BACKPHOTO_TYPE   GLOBAL         # can be GLOBAL or LOCAL\n",
    "\n",
    "#     #------------------------------ Check Image ----------------------------------\n",
    "\n",
    "#     CHECKIMAGE_TYPE  MODELS, -MODELS, -BACKGROUND          # can be NONE, BACKGROUND, BACKGROUND_RMS,\n",
    "#                                     # MINIBACKGROUND, MINIBACK_RMS, -BACKGROUND,\n",
    "#                                     # FILTERED, OBJECTS, -OBJECTS, SEGMENTATION,\n",
    "#                                     # or APERTURES\n",
    "#     CHECKIMAGE_NAME  prof.fits,subprof.fits,orig.fits     # Filename for the check-image\n",
    "\n",
    "#     #--------------------- Memory (change with caution!) -------------------------\n",
    "\n",
    "#     MEMORY_OBJSTACK  3000           # number of objects in stack\n",
    "#     MEMORY_PIXSTACK  300000         # number of pixels in stack\n",
    "#     MEMORY_BUFSIZE   1024           # number of lines in buffer\n",
    "\n",
    "#     #----------------------------- Miscellaneous ---------------------------------\n",
    "\n",
    "#     VERBOSE_TYPE     NORMAL         # can be QUIET, NORMAL or FULL\n",
    "#     WRITE_XML        N              # Write XML file (Y/N)?\n",
    "#     XML_NAME         sex.xml        # Filename for XML output\n",
    "#     '''\n",
    "#     #-----------------------------------------------------------------------------------------\n",
    "\n",
    "#     param_content = '''# Catalog parameters \n",
    "#     X_IMAGE\n",
    "#     Y_IMAGE\n",
    "#     A_IMAGE\n",
    "#     B_IMAGE\n",
    "#     FLUX_PSF\n",
    "#     CLASS_STAR\n",
    "#     '''\n",
    "\n",
    "#     #-----------------------------------------------------------------------------------------\n",
    "\n",
    "#     with open(dst_dir + 'LF_01.sex', 'w') as f:\n",
    "#         f.write(sex_content)\n",
    "\n",
    "#     with open(dst_dir + 'LF_01.param', 'w') as f:\n",
    "#         f.write(param_content)\n",
    "\n",
    "#     conv_src_dir = default_SExtractor_files_dir + 'default.conv'\n",
    "#     nnw_src_dir  = default_SExtractor_files_dir + 'default.nnw'\n",
    "#     psf_src_dir  = default_SExtractor_files_dir + 'default.psf'\n",
    "\n",
    "#     subprocess.check_call(['cp', conv_src_dir, dst_dir + 'default.conv'])\n",
    "#     subprocess.check_call(['cp', nnw_src_dir,  dst_dir + 'default.nnw'])\n",
    "#     subprocess.check_call(['cp', psf_src_dir,  dst_dir + 'default.psf'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# generate_sex_files(dst_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/')\n",
    "\n",
    "# single_crop('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/COSMOS_ASC_055.fits', 3000, 12500, cutout_shape=(1000, 1000))\n",
    "\n",
    "# SExtract_v2('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/', fits_name='acs_I_030mas_075_sci.fits')\n",
    "\n",
    "# create_raws('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/', 'acs_I_030mas_075_sci.fits', cutout_shape=(100, 100), cat_name='cat_star.csv')\n",
    "\n",
    "# create_raws('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/', 'acs_I_030mas_075_sci.fits', cutout_shape=(200, 200), cat_name='selected02_cat .csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # the following are 60 elliptical galaxies from COSMOS 75\n",
    "# elliptical_indices = [7, 17, 19, 20, 24, 34, 37, 51, 55, 60, 63, 73, 75, 76, 77, 78, 83, 84, 104, 108, 109, 111, 112, 116, 121, 145, 165, 234, 235, 250, 263, 267, 272, 316, 345, 368, 472, 474, 481, 501, 511, 513, 516, 526, 559, 570, 573, 578, 589, 592, 598, 604, 607, 622, 650, 664, 803, 804, 842, 847]\n",
    "\n",
    "# src_folder = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/raws/'\n",
    "# dst_folder = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/ellipticals/'\n",
    "\n",
    "# names = list_file_names(src_folder, shuffle_names=False)\n",
    "# index = 7\n",
    "# elliptical_names = []\n",
    "# for index in elliptical_indices:\n",
    "#     for name in names:\n",
    "#         if ('_i' + str(index) + '_') in name:\n",
    "#             elliptical_names.append(name)\n",
    "            \n",
    "# for i, name in enumerate(elliptical_names):\n",
    "#     src_dir = src_folder + name\n",
    "#     dst_dir = dst_folder + 'raw_75_' + str(elliptical_indices[i]).zfill(2) + '.fits'\n",
    "#     subprocess.call('cp -u ' + src_dir + ' ' + dst_dir, shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # copying elliptical galaxies from COSMOS 55\n",
    "# good_sources = [234,224,348,359,432,445,454,326,502,516,579,571,637,709,961,1069,1245,1218,1244,1210,1199,1444,1420,1582,1904,1946,2332,2458,2488,2596,2675,2793,2942,2966,2960,3275,3659,3778,3805]\n",
    "\n",
    "# # i = 0\n",
    "\n",
    "# def copy_raws(index,\n",
    "#         input_folder_dir  = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/lenses/',\n",
    "#         output_folder_dir = './COSMOS_ASC_055/good_lenses/'):\n",
    "#     global i\n",
    "#     src_dir = input_folder_dir  + 'raw_' + str(index).zfill(4) + '.fits'\n",
    "#     dst_dir = output_folder_dir + 'raw_55_' + str(i).zfill(2) + '.fits'\n",
    "#     #!cp -u $src_dir $dst_dir\n",
    "#     subprocess.call('cp -u ' + src_dir + ' ' + dst_dir, shell=True)\n",
    "    \n",
    "# for j in good_sources:\n",
    "#     copy_raws(index=j, input_folder_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/raws/', output_folder_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/ellipticals/')\n",
    "#     i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# src_folder = '/home/USER/Desktop/Data_for_LF/ellipticals/small_lenses/'\n",
    "# dst_folder = '/home/USER/Desktop/Data_for_LF/ellipticals/small_lenses_x8/'\n",
    "\n",
    "# names = list_file_names(src_folder)\n",
    "\n",
    "# for name in names:\n",
    "#     transform_fits(src_folder, name, dst_folder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# auto_merge('/home/USER/Desktop/Data_for_LF/ellipticals/raws/', \n",
    "#            '/home/USER/Desktop/Data_for_LF/ellipticals/rings2/', \n",
    "#            '/home/USER/Desktop/Data_for_LF/ellipticals/lenses1/', \n",
    "#            coef_range=(0.3, 0.8))\n",
    "\n",
    "# list_and_label_files('/home/USER/Desktop/Data_for_LF/ellipticals/lenses/', 'lensed')\n",
    "\n",
    "# list_and_label_files('/home/USER/Desktop/Data_for_LF/ellipticals/raws_x8/', 'unlensed')\n",
    "\n",
    "# list_and_label_files('/home/USER/Desktop/Data_for_LF/ellipticals/small_lenses_x8/', 'lensed')\n",
    "\n",
    "# merge_two_csv_files('/home/USER/Desktop/Data_for_LF/ellipticals/lenses.csv', '/home/USER/Desktop/Data_for_LF/ellipticals/raws_x8.csv', '/home/USER/Desktop/Data_for_LF/ellipticals/first.csv')\n",
    "\n",
    "# merge_two_csv_files('/home/USER/Desktop/Data_for_LF/ellipticals/first.csv', '/home/USER/Desktop/Data_for_LF/ellipticals/small_lenses_x8.csv', '/home/USER/Desktop/Data_for_LF/ellipticals/elliptical_train.csv')\n",
    "\n",
    "# merge_two_csv_files('/home/USER/Desktop/Data_for_LF/ellipticals/elliptical_train.csv', \n",
    "#                     '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/train_02.csv', \n",
    "#                     '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/train_03.csv')\n",
    "\n",
    "# merge_two_csv_files('/home/USER/Desktop/Data_for_LF/ellipticals/elliptical_train.csv', \n",
    "#                     '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/unfaithful_raws.csv', \n",
    "#                     '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/train_04.csv')\n",
    "\n",
    "# generate_sex_files(dst_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_065/')\n",
    "\n",
    "# SExtract_v2('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_065/', fits_name='acs_I_030mas_065_sci.fits')\n",
    "\n",
    "\n",
    "# merge_two_csv_files('/home/USER/Desktop/Data_for_LF/ellipticals/elliptical_train.csv', \n",
    "#                     '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/unfaithful_raws_02.csv', \n",
    "#                     '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/train_05.csv')\n",
    "\n",
    "# create_raws('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_065/', 'acs_I_030mas_065_sci.fits', cutout_shape=(200, 200), cat_name='organized_cat.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# input_txt_dir  = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_065/cat.txt'\n",
    "# output_csv_dir = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_065/cat.csv'\n",
    "\n",
    "# data = np.loadtxt(cat_dir, comments='#')\n",
    "# np.set_printoptions(suppress=True)\n",
    "# np.savetxt(output_csv_dir, data, fmt='%f', delimiter=',')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# txt_to_csv(input_txt_dir='/home/USER/Desktop/Data_for_LF/COSMOS_ASC_065/cat.txt')\n",
    "\n",
    "# list_and_label_files('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_075/raws/', 'unlensed')\n",
    "\n",
    "# list_and_label_files('/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/lens_a2r/', 'lensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Plot LF Performance\n",
    "# data = np.array([24,39,43,48,77,91,95,117,137,155,177,189,219,361])/3506*100\n",
    "# plt.plot([1,2,3,4,5,6,7,8,9,10,11,12,13,14], data, 'o')\n",
    "# plt.axis([0, 16, 0, 11])\n",
    "# plt.xticks(np.arange(1,15))\n",
    "# plt.yticks(np.array([0,1,2,3,4,5,6,10]))\n",
    "\n",
    "# plt.title('Lens Detection Performance of LensingFlow 16Mar2017 v01')\n",
    "# plt.xlabel('Real Lens Index')\n",
    "# plt.ylabel('Rank Ratio (%)')\n",
    "\n",
    "# plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# single_crop('/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/0001_150.07667000_2.64583000_acs_I_mosaic_30mas_sci.fits', 210, 273)\n",
    "\n",
    "# generate_sex_files('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/')\n",
    "\n",
    "# SExtract_v2('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/', 'acs_I_030mas_098_sci.fits')\n",
    "\n",
    "# txt_to_csv('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/cat.txt')\n",
    "\n",
    "\n",
    "# create_raws('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/acs_I_030mas_098_sci.fits', cat_name='cat_starless.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Steps I'm taking for 098:\n",
    "1. Downloaded the file and unzipped it in a folder named COSMOS_ASC_098\n",
    "2. SExtract('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/', 'acs_I_030mas_098_sci.fits')\n",
    "3. Removed rows with high star parameter (i.e. last column) from the generated cat.csv and name it cat_starless.csv.\n",
    "4. create_raws('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/acs_I_030mas_098_sci.fits', cat_name='cat_starless.csv')\n",
    "5. list_and_label_files('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/', 'unlensed')\\\n",
    "6. Add cosmos lenses to raws.csv and call it raws_plus.csv:\n",
    "\n",
    "```python\n",
    "merge_two_csv_files('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/raws.csv',\n",
    "                    '/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/lens_a2r.csv', \n",
    "                    '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_098/raws_plus.csv')\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# # Create downsampled train set\n",
    "# dir1 = '/home/USER/Desktop/Data_for_LF/ellipticals/downsampled_lenses/'\n",
    "# dir2 = '/home/USER/Desktop/Data_for_LF/ellipticals/downsampled_raws_x8/'\n",
    "# dir3 = '/home/USER/Desktop/Data_for_LF/ellipticals/downsampled_small_lenses_x8/'\n",
    "# dir4 = '/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/downsampled_raws/'\n",
    "\n",
    "# dir1 = list_and_label_files(dir1, 'lensed')\n",
    "# dir2 = list_and_label_files(dir2, 'unlensed')\n",
    "# dir3 = list_and_label_files(dir3, 'lensed')\n",
    "# dir4 = list_and_label_files(dir4, 'unlensed')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# dir12 = merge_two_csv_files(dir1, dir2, '/home/USER/Desktop/12.csv')\n",
    "# dir123 = merge_two_csv_files(dir12, dir3, '/home/USER/Desktop/123.csv')\n",
    "# dir1234 = merge_two_csv_files(dir123, dir4, '/home/USER/Desktop/1234.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# list_and_label_files('/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/downsampled_lens_a2r/', 'lensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# file_dir = '/home/USER/Desktop/Data_for_LF/ellipticals/lenses/lens_13.fits'\n",
    "\n",
    "# # 1.\n",
    "# image_data = fits.getdata(file_dir)\n",
    "\n",
    "# # 2.\n",
    "# tot = np.sum(image_data)\n",
    "# print(tot)\n",
    "\n",
    "# mean       = np.average(image_data)\n",
    "# print(mean)\n",
    "\n",
    "# maximum = np.max(image_data)\n",
    "# print(maximum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize_01(image_data, cap=False):\n",
    "    '''\n",
    "    1. Gets FITS data from the given file_dir.\n",
    "    2. Evaluates mean pixel value.\n",
    "    3. Subtracts the mean from all pixels.\n",
    "    4. Divide by the largest pixel magnitude\n",
    "    5. returns the updated image_data.'''\n",
    "    \n",
    "    # 1.\n",
    "    # image_data = fits.getdata(file_dir)\n",
    "    \n",
    "    # 2.\n",
    "    mean       = np.mean(image_data)\n",
    "    \n",
    "    # 3. \n",
    "    image_data = image_data - mean\n",
    "    \n",
    "    # 4.\n",
    "    if cap is True:\n",
    "        sigma = np.std(image_data)\n",
    "        image_data = np.minimum(image_data, +3*sigma)\n",
    "        image_data = np.maximum(image_data, -3*sigma)\n",
    "    \n",
    "    norm_coef = np.std(image_data)\n",
    "\n",
    "    if norm_coef != 0:\n",
    "        image_data = image_data/norm_coef\n",
    "    \n",
    "    # 5.\n",
    "    return image_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auto_normalize_01(input_fits_folder_dir, output_fits_folder_dir):\n",
    "    names = list_file_names(input_fits_folder_dir)\n",
    "    subprocess.call(['mkdir', output_fits_folder_dir])\n",
    "    \n",
    "    for index, name in enumerate(names):\n",
    "        image_org = fits.getdata(input_fits_folder_dir + name)\n",
    "        image = normalize_01(image_org)\n",
    "        save_fits(image, save_to=output_fits_folder_dir + name)\n",
    "        \n",
    "    !nautilus $output_fits_folder_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge_two_fits_01(dir_1, dir_2, coef_2, output_dir):\n",
    "    '''Gets the normalized image data of the two fits files specified at dir_1 and dir_2.\n",
    "    Multiplies the second one by coef_2 and adds it to the first one.\n",
    "    Saves the normalized result at output_dir.\n",
    "    Note: This will overwrite output_dir if it is already an existing file.'''\n",
    "    \n",
    "    image_1 = get_fits_data(dir_1, method='divide by max')\n",
    "    image_2 = get_fits_data(dir_2, method='divide by max')\n",
    "    \n",
    "    # If the dimension is right, add and normalize\n",
    "    if image_1.shape == image_2.shape:\n",
    "        image_3 = image_1 + coef_2 * image_2\n",
    "        image_3 = normalize_01(image_3)\n",
    "    else:\n",
    "        print('Milad: Different fits dimensions in merge_fits(...).')\n",
    "    \n",
    "    # Save to output_dir\n",
    "    save_fits(image_3, output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def merge_two_fits_02(dir_1, dir_2, coef_2, output_dir):\n",
    "    '''Gets the normalized image data of the two fits files specified at dir_1 and dir_2.\n",
    "    Multiplies the second one by coef_2 and adds it to the first one.\n",
    "    Saves the normalized result at output_dir.\n",
    "    Note: This will overwrite output_dir if it is already an existing file.'''\n",
    "    \n",
    "    image_1 = get_fits_data(dir_1, method='norm')\n",
    "    image_2 = get_fits_data(dir_2, method='norm')\n",
    "    \n",
    "    # If the dimension is right, add and normalize\n",
    "    if image_1.shape == image_2.shape:\n",
    "        image_3 = image_1 + coef_2 * image_2\n",
    "#         image_3 = normalize_01(image_3)\n",
    "    else:\n",
    "        print('Milad: Different fits dimensions in merge_fits(...).')\n",
    "    \n",
    "    # Save to output_dir\n",
    "    save_fits(image_3, output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auto_merge_01(raws_csv_dir, arc_csv_dir, output_folder_dir, num_merges=None, coef_range=(0.2, 0.8)):\n",
    "    raw_lines  = read_csv(raws_csv_dir, 'list of rows')\n",
    "    num_raws   = len(raw_lines)\n",
    "    \n",
    "    arc_lines  = read_csv(arc_csv_dir, 'list of rows')\n",
    "    num_rings  = len(arc_lines)\n",
    "        \n",
    "    subprocess.call(['mkdir', output_folder_dir])    \n",
    "    \n",
    "    if num_merges is None:\n",
    "        num_merges = min(num_raws, num_rings)\n",
    "    \n",
    "    for i in range(num_merges):\n",
    "        # Randomize ring brightness\n",
    "        coef = random.uniform(*coef_range)\n",
    "        # Merge and save\n",
    "        merge_two_fits_02(dir_1=raw_lines[i][0] + raw_lines[i][1], \n",
    "                          dir_2=arc_lines[i][0] + arc_lines[i][1], \n",
    "                          coef_2=coef, \n",
    "                          output_dir=output_folder_dir + 'lens_' + str(i) + '.fits')\n",
    "    \n",
    "    !nautilus $output_folder_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def auto_merge_02(raws_folder_dir, ring_folder_dir, output_folder_dir, num_merges=None, coef_range=(0.2, 0.8)):\n",
    "    raw_names  = list_file_names(raws_folder_dir)\n",
    "    num_raws   = len(raw_names)\n",
    "    \n",
    "    ring_names = list_file_names(ring_folder_dir)\n",
    "    num_rings  = len(ring_names)\n",
    "        \n",
    "    subprocess.call(['mkdir', output_folder_dir])    \n",
    "    \n",
    "    if num_merges is None:\n",
    "        num_merges = min(num_raws, num_rings)\n",
    "    \n",
    "    for i in range(num_merges):\n",
    "        # Randomize ring brightness\n",
    "        coef = random.uniform(*coef_range)\n",
    "        # Merge and save\n",
    "        merge_two_fits_02(dir_1=raws_folder_dir + raw_names[i], dir_2=ring_folder_dir + ring_names[i], coef_2=coef, output_dir=output_folder_dir + 'lens_' + str(i) + '.fits')\n",
    "    \n",
    "    !nautilus $output_folder_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# auto_normalize_01('/home/USER/Desktop/Data_for_LF/faints/ellipticals/', \n",
    "#                   '/home/USER/Desktop/Data_for_LF/faints/ellipticals_normalized/')\n",
    "\n",
    "# auto_merge_01('/home/USER/Desktop/Data_for_LF/faints/ellipticals/', \n",
    "#               '/home/USER/Desktop/Data_for_LF/faints/rings_normalized/',\n",
    "#               '/home/USER/Desktop/Data_for_LF/faints/elliptical_lenses_normalized/', \n",
    "#               coef_range=(0.01,0.03))\n",
    "\n",
    "# [d for d in os.listdir('/home/USER/Desktop/Data_for_LF/faints/') if os.path.isdir(os.path.join('/home/USER/Desktop/Data_for_LF/faints/', d))]\n",
    "\n",
    "# dir1 = list_and_label_files('/home/USER/Desktop/Data_for_LF/faints/raws_normalized/', file_label='unlensed')\n",
    "# dir2 = list_and_label_files('/home/USER/Desktop/Data_for_LF/faints/ellipticals_normalized/', file_label='unlensed')\n",
    "# dir3 = list_and_label_files('/home/USER/Desktop/Data_for_LF/faints/elliptical_lenses_normalized/', file_label='lensed')\n",
    "\n",
    "# merge_two_csv_files(dir2, dir3, '/home/USER/Desktop/Data_for_LF/faints/train_01.csv')\n",
    "\n",
    "# auto_normalize_01('/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/lens_a2r_x8/',\n",
    "#                   '/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/lens_a2r_x8_normalized/')\n",
    "\n",
    "# list_and_label_files('/home/USER/Dropbox/LensingFlow/LensingFlow_Dropbox/Extras/lens_a2r_x8_normalized/', 'lensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_073/COSMOS_ASC_073.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_074/COSMOS_ASC_074.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_075/COSMOS_ASC_075.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_076/COSMOS_ASC_076.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_077/COSMOS_ASC_077.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_078/COSMOS_ASC_078.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_079/COSMOS_ASC_079.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_080/COSMOS_ASC_080.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_081/COSMOS_ASC_081.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_085/COSMOS_ASC_085.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_086/COSMOS_ASC_086.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_087/COSMOS_ASC_087.fits', cat_name='cat.csv')\n",
    "# create_raws('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_088/COSMOS_ASC_088.fits', cat_name='cat.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# csv1 = read_csv('/home/USER/Desktop/Data_for_LF/ellipticals/lenses_x8.csv', 'list of rows')\n",
    "# csv2 = read_csv('/home/USER/Desktop/Data_for_LF/ellipticals/small_lenses_x8.csv', 'list of rows')\n",
    "# csv3 = read_csv('/home/USER/Desktop/Data_for_LF/COSMOS_ASC_055/good_lenses_x8.csv', 'list of rows')\n",
    "\n",
    "# csv_lens = []\n",
    "# csv_lens.extend(csv1)\n",
    "# csv_lens.extend(csv2)\n",
    "# csv_lens.extend(csv3);\n",
    "\n",
    "# write_csv(csv_lens, '/home/USER/Desktop/Data_for_LF/lenses.csv', 'list of rows')\n",
    "\n",
    "\n",
    "#13, 14, 26,25,21.33,32,20\n",
    "\n",
    "# csv1 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_013/raws.csv', 'list of rows')\n",
    "# csv2 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_014/raws.csv', 'list of rows')\n",
    "# csv3 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_026/raws.csv', 'list of rows')\n",
    "# csv4 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_025/raws.csv', 'list of rows')\n",
    "# csv5 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_021/raws.csv', 'list of rows')\n",
    "# csv6 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_033/raws.csv', 'list of rows')\n",
    "# csv7 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_032/raws.csv', 'list of rows')\n",
    "# csv8 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_020/raws.csv', 'list of rows')\n",
    "\n",
    "# csv9 = read_csv('/media/USER/black_book/USER/COSMOS/COSMOS_ASC_053/raws.csv', 'list of rows')\n",
    "\n",
    "# csv_merged = []\n",
    "# csv_merged.extend(csv1)\n",
    "# csv_merged.extend(csv2)\n",
    "# csv_merged.extend(csv3)\n",
    "# csv_merged.extend(csv4)\n",
    "# csv_merged.extend(csv5)\n",
    "# csv_merged.extend(csv6)\n",
    "# csv_merged.extend(csv7)\n",
    "# csv_merged.extend(csv8)\n",
    "# csv_merged.extend(csv9)\n",
    "\n",
    "# len(csv_merged)\n",
    "# write_csv(csv_merged, '/home/USER/Desktop/Data_for_LF/raws.csv', 'list of rows')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for folder in folder_dirs:\n",
    "#     list_and_label_files(folder+'raws/', 'unlensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# multi_ds9('/home/USER/Desktop/KLF_11Apr2017_V01_COSMOS_Scan.csv', range(0,100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# multi_ds9('/home/USER/Desktop/KLF_11Apr2017_V01_COSMOS_Scan (copy).csv', range(2000,2200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# for i in range(200,205):\n",
    "#     multi_ds9('/home/USER/Desktop/top_COSMOS_sources.csv', range(i*100,(i+1)*100))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# for i in range(46, 80):\n",
    "#     multi_ds9('/home/USER/Desktop/KLF_11Apr2017_V01_Phase2_16Apr2017_onefold.csv', range(i*50,(i+1)*50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def search_csv():\n",
    "    '''Lists directories of sources given tile number and index number.'''\n",
    "    \n",
    "    input_dir = '/home/USER/Desktop/top_COSMOS_sources.csv'\n",
    "    input_dir = '/media/USER/black_book/USER/COSMOS/KLF_11Apr2017_V01_COSMOS_Scan.csv'\n",
    "    output_dir = '/home/USER/Desktop/selection.csv'\n",
    "\n",
    "    all_names = read_csv(input_dir, 'list of columns')[1]\n",
    "    lines     = read_csv(input_dir, 'list of rows')\n",
    "    selected_lines = []\n",
    "    selections = [[87, 2666], [87, 1137], [112, 1395], [90, 2233], [51, 2300], [45, 1846], [114, 172], [89, 195], [52, 4216], [79, 792], [102, 1057], [90, 4053], [32, 1572], [73, 4053], [41, 1062], [78, 3519], [43, 2356], [41, 1496], [102, 4229], [75, 1200], [74, 3939], [88, 1825], [32, 1724], [64, 1932], [78, 1286], [87, 3791], [66, 2205], [67, 3374], [63, 3336], [53, 3219], [50, 2921], [39, 3182], [40, 1613], [19, 240], [38, 1831], [32, 130], [74, 4051], [41, 2204], [79, 675], [101, 613], [80, 612], [99, 2352], [97, 302], [66, 3951], [57, 2209], [88, 2950], [100, 1212], [113, 1552], [75, 1200], [28, 937], [114, 862], [98, 2663], [99, 3799], [104, 3540], [32, 130], [89, 819], [116, 534]]\n",
    "    selections = [[87, 2666], [87, 1137], [112, 1395], [90, 2233], [45, 1846], [114, 172], [52, 4216], [79, 792], [102, 1057], [32, 1572], [73, 4053], [41, 1062], [41, 1496], [102, 4229], [75, 1200], [74, 3939], [88, 1825], [32, 1724], [64, 1932], [78, 1286], [87, 3791], [63, 3336], [53, 3219], [50, 2921], [39, 3182], [40, 1613], [19, 240], [38, 1831], [32, 130], [74, 4051], [41, 2204], [79, 675], [101, 613], [80, 612], [99, 2352], [97, 302], [66, 3951], [57, 2209], [88, 2950], [100, 1212], [113, 1552], [75, 1200], [28, 937], [114, 862], [98, 2663], [104, 3540], [116, 534]]\n",
    "    for tile_num, source_index in selections:\n",
    "        # tile_num = selections[0][i]\n",
    "        # source_index = selections[1][i]\n",
    "        partial_name = 'COSMOS_ASC_' + '{:03}'.format(tile_num) + '_raws_i' + str(source_index) + '_'\n",
    "        full_name    = [s for s in all_names if partial_name in s]\n",
    "        if len(full_name) != 0:\n",
    "            line_index   = all_names.index(full_name[0])\n",
    "            selected_lines.append(lines[line_index])\n",
    "        else:\n",
    "            print(partial_name + ' was not found.')\n",
    "    \n",
    "    for line in selected_lines:\n",
    "        src = line[0] + line[1]\n",
    "        dst = '/home/USER/Desktop/lenses0/' + line[1]\n",
    "        !cp $src $dst\n",
    "    \n",
    "#     write_csv(selected_lines, output_dir, 'list of rows')\n",
    "#     print(str(len(selected_lines)) + ' was saved at: ' + output_dir)\n",
    "    \n",
    "search_csv()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "multi_ds9('/home/USER/Desktop/KLF_11Apr2017_V04_Scan.csv', range(0, 46))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(20,50):\n",
    "    multi_ds9('/home/USER/Desktop/KLF_11Apr2017_V04.2_Scan.csv', range(100*i, 100*i+100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Generate ROC diagram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "multi_ds9('/home/USER/Desktop/top_phaseIII_ScanIV.csv',range(600,700))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "auto_transform('/media/USER/black_book/USER/lenstool_kitchen_02/ellipticals/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "auto_merge_02('/media/USER/black_book/USER/ellipticals/ellipticals_x8/',\n",
    "              '/media/USER/black_book/USER/lenstool_kitchen_02/rings/',\n",
    "              '/media/USER/black_book/USER/ellipticals/lenses/',coef_range=(0.05,0.4))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# # Rename some lenses\n",
    "# src_dir = '/home/USER/Desktop/Data_for_LF/ellipticals_2/ellipticals_noah/'\n",
    "# dst_dir = '/home/USER/Desktop/Data_for_LF/ellipticals_2/noah/'\n",
    "# names = list_file_names(src_dir)\n",
    "\n",
    "# for i, name in enumerate(names):\n",
    "#     subprocess.check_call('cp -u ' + src_dir + name + ' ' + dst_dir + str(i) + '.fits', shell=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "auto_transform('/media/USER/black_book/USER/ellipticals/lenses/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "list_and_label_files('/media/USER/black_book/USER/ellipticals/lenses_x8/', 'lensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "auto_transform('/media/USER/black_book/USER/lenstool_kitchen_02/rings/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "list_and_label_files('/media/USER/black_book/USER/lenstool_kitchen_02/rings_x8/', 'arc')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "auto_merge_01('/home/USER/Desktop/elliptical_raws.csv','/media/USER/black_book/USER/lenstool_kitchen_02/rings_x8.csv','/media/USER/black_book/USER/ellipticals/more_lenses/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "list_and_label_files('/media/USER/black_book/USER/ellipticals/more_lenses/','lensed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "dir = '/home/USER/Desktop/lens_01.fits'\n",
    "image_data = fits.getdata(dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_coord(fits_dir, pixel=(100, 100)):\n",
    "    '''Retrives the RA (deg) and Dec (deg) of the specified pixel of the FITS file located at fits_dir.'''\n",
    "    f = fits.open(fits_dir)\n",
    "    w = wcs.WCS(f[0].header)\n",
    "    coords = w.wcs_pix2world(100, 100, 1)\n",
    "    ra  = coords[0]+0.\n",
    "    dec = coords[1]+0.\n",
    "    return float(\"{0:.4f}\".format(ra)), float(\"{0:.4f}\".format(dec))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data = read_csv(\"/home/USER/Desktop/top_ranks_with_coords.csv\", \"list of rows\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for i in range(len(data)):\n",
    "    fits_dir = data[i][0] + data[i][1]\n",
    "    coords = get_coord(fits_dir)\n",
    "    data[i].append(coords[0])\n",
    "    data[i].append(coords[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "write_csv(data,\"/home/USER/Desktop/top_ranks_with_coords.csv\", \"list of rows\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "multi_ds9('/media/USER/black_book/USER/COSMOS/KLF_11Apr2017_V01_COSMOS_Scan.csv',range(400,500))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}