create_ricord_dataset/create_ricord_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import cv2\n",
    "import glob\n",
    "import pydicom\n",
    "from pydicom.pixel_data_handlers import apply_modality_lut, apply_voi_lut\n",
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Helper functions\"\"\"\n",
    "def _extract_data(df_row):\n",
    "    return df_row['Anon MRN'], df_row['Anon TCIA Study Date'], df_row['Anon Exam Description'], df_row['Anon Study UID']\n",
    "\n",
    "\n",
    "def load_ricord_metadata(ricord_meta_file):\n",
    "    df = pd.read_excel(ricord_meta_file, sheet_name='CR Pos - TCIA Submission')\n",
    "    ricord_metadata = []\n",
    "    for index, row in df.iterrows():\n",
    "        ricord_metadata.append(_extract_data(row))\n",
    "    return ricord_metadata\n",
    "\n",
    "def make_ricord_dict(ricord_data_set_file):\n",
    "    \"\"\"Loads bboxes from the given text file\"\"\"\n",
    "    ricord_dict = {}\n",
    "    with open(ricord_data_set_file, 'r') as f:\n",
    "        for line in f.readlines():\n",
    "            # Values after file name are crop dimensions\n",
    "            if(len(line.split()) > 1):\n",
    "                fname, xmin, ymin, xmax, ymax = line.rstrip('\\n').split()\n",
    "                bbox = tuple(int(c) for c in (xmin, ymin, xmax, ymax))\n",
    "                ricord_dict[fname] = bbox\n",
    "            else:\n",
    "                fname = line.rstrip('\\n')\n",
    "                ricord_dict[fname] = None\n",
    "                \n",
    "    return ricord_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "RICORD data requires some preprocessing before splitting into test/train. \n",
    "Some images contain padding and some images are unusable. \n",
    "ricord_data_set.txt contains the name of usable images along with bounding box dimensions if needed. \n",
    "\n",
    "This cell crops the DICOM according to dimensions in ricord_data_set.txt and saves the image \n",
    "as png format in out_dir.\n",
    "\n",
    "DICOM_images and \"MIDRC-RICORD-1c Clinical Data Jan 13 2021 .xlsx\" need to be downloaded from\n",
    "https://wiki.cancerimagingarchive.net/pages/viewpage.action?pageId=70230281 before running this cell\n",
    "\"\"\"\n",
    "\n",
    "ricord_dir = 'path/to/DICOM_images'\n",
    "ricord_meta_file = 'path/to/MIDRC-RICORD-1c Clinical Data Jan 13 2021 .xlsx'\n",
    "\n",
    "out_dir = 'ricord_images'\n",
    "ricord_set_file = 'ricord_data_set.txt'\n",
    "\n",
    "os.makedirs(out_dir, exist_ok=True)\n",
    "\n",
    "ricord_dict = make_ricord_dict(ricord_set_file)\n",
    "\n",
    "metadata = load_ricord_metadata(ricord_meta_file)\n",
    "file_count = 0\n",
    "for mrn, date, desc, uid in metadata:\n",
    "    date = date.strftime('%m-%d-%Y')\n",
    "    uid = uid[-5:]\n",
    "    study_dir = os.path.join(ricord_dir, 'MIDRC-RICORD-1C-{}'.format(mrn), '*-{}'.format(uid))\n",
    "    dcm_files = sorted(glob.glob(os.path.join(study_dir, '*', '*.dcm')))\n",
    "    for i, dcm_file in enumerate(dcm_files):\n",
    "        # Create output path and check if image is to be included\n",
    "        out_fname = 'MIDRC-RICORD-1C-{}-{}-{}.png'.format(mrn, uid, i)\n",
    "        out_path = os.path.join(out_dir, out_fname)\n",
    "\n",
    "        if out_fname not in ricord_dict:\n",
    "            continue\n",
    "\n",
    "        # Load DICOM image\n",
    "        ds = pydicom.dcmread(dcm_file)\n",
    "\n",
    "        # Verify orientation\n",
    "        if ds.ViewPosition != 'AP' and ds.ViewPosition != 'PA':\n",
    "            print('Image from MRN-{} Date-{} UID-{} in position {}'.format(mrn, date, uid, ds.ViewPosition))\n",
    "            continue\n",
    "\n",
    "        # Apply transformations if required\n",
    "        if ds.pixel_array.dtype != np.uint8:\n",
    "            # Apply LUT transforms\n",
    "            arr = apply_modality_lut(ds.pixel_array, ds)\n",
    "            if arr.dtype == np.float64 and ds.RescaleSlope == 1 and ds.RescaleIntercept == 0:\n",
    "                arr = arr.astype(np.uint16)\n",
    "            arr = apply_voi_lut(arr, ds)\n",
    "            arr = arr.astype(np.float64)\n",
    "\n",
    "            # Normalize to [0, 1]\n",
    "            arr = (arr - arr.min())/arr.ptp()\n",
    "\n",
    "            # Invert MONOCHROME1 images\n",
    "            if ds.PhotometricInterpretation == 'MONOCHROME1':\n",
    "                arr = 1. - arr\n",
    "\n",
    "            # Convert to uint8\n",
    "            image = np.uint8(255.*arr)\n",
    "        else:\n",
    "            # Invert MONOCHROME1 images\n",
    "            if ds.PhotometricInterpretation == 'MONOCHROME1':\n",
    "                image = 255 - ds.pixel_array\n",
    "            else:\n",
    "                image = ds.pixel_array\n",
    "\n",
    "        # Crop if necessary\n",
    "        bbox = ricord_dict[out_fname]\n",
    "        if bbox is not None:\n",
    "            image = image[bbox[1]:bbox[3], bbox[0]:bbox[2]]\n",
    "\n",
    "        # Save image\n",
    "        cv2.imwrite(out_path, image)\n",
    "        file_count += 1\n",
    "print('Created {} files'.format(file_count))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "andy.zhao",
   "language": "python",
   "name": "andy.zhao"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}