From a5073e7616eed7c5ef62ef8ec1eee5d0ca0164e5 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:54:02 +0200 Subject: [PATCH 01/25] add perens mri --- .../perens_stereotaxic_mri_mouse.py | 385 ++++++++++++++++++ 1 file changed, 385 insertions(+) create mode 100644 brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py new file mode 100644 index 00000000..e310c3cd --- /dev/null +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -0,0 +1,385 @@ +import sys +sys.path.append(r"/home/harryc/github/brainglobe-atlasapi/") + + +__version__ = "3" +import os +import json +import multiprocessing as mp +import tarfile +import time +from pathlib import Path +import requests +import numpy as np +import pandas as pd +import SimpleITK as sitk +from rich.progress import track +import py7zr +from brainglobe_atlasapi import utils +from brainglobe_atlasapi.atlas_generation.mesh_utils import ( + Region, + create_region_mesh, +) +from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data +from brainglobe_atlasapi.structure_tree_util import get_structures_tree + +PARALLEL = True # disable parallel mesh extraction for easier debugging + + +### Additional functions ##################################################### + + +############################################################################## +def get_id_from_acronym(df, acronym): + """ + Get Allen's brain atlas ID from brain region acronym(s) + + Call: + get_id_from_acronym(df, acronym) + + Args: + df (pandas dataframe): + atlas table file [see atlas.load_table()] + acronym (string or list of strings): brain region acronym(s) + + Returns: + ID (int or list of ints): + brain region ID(s) corresponding to input acronym(s) + """ + + # create as list if necessary + if not isinstance(acronym, list): + acronym = [acronym] + + if len(acronym) > 1: + ID_list = [] + for acro in acronym: + ID = df["id"][df["acronym"] == acro].item() + ID_list.append(ID) + return ID_list + else: + return df["id"][df["acronym"] == acronym[0]].item() + + # return df['id'][df['acronym'] == acronym].item() # OLD VERSION + + +def get_acronym_from_id(df, ID): + """ + Get Allen's brain atlas acronym from brain region ID(s) + + Call: + get_acronym_from_ID(df, acronym) + + Args: + df (pandas dataframe): atlas table dataframe [see atlas.load_table()] + ID (int or list of int): brain region ID(s) + + Returns: + acronym (string or list of strings): + brain region acronym(s) corresponding to input ID(s) + """ + + # create as list if necessary + if not isinstance(ID, list): + ID = [ID] + + if len(ID) > 1: + acronym_list = [] + for id in ID: + acronym = df["acronym"][df["id"] == id].item() + acronym_list.append(acronym) + return acronym_list + else: + return df["acronym"][df["id"] == ID[0]].item() + + +def tree_traverse_child2parent(df, child_id, ids): + parent = df["parent_id"][df["id"] == child_id].item() + + if not np.isnan(parent): + id = df["id"][df["id"] == parent].item() + ids.append(id) + tree_traverse_child2parent(df, parent, ids) + return ids + else: + return ids + + +def get_all_parents(df, key): + """ + Get all parent IDs/acronyms in Allen's brain atlas hierarchical structure' + + Call: + get_all_children(df, key) + + Args: + df (pandas dataframe) : atlas table dataframe [see atlas.load_table()] + key (int/string) : atlas region ID/acronym + + Returns: + parents (list) : brain region acronym corresponding to input ID + """ + + if isinstance(key, str): # if input is acronym convert to ID + list_parent_ids = tree_traverse_child2parent( + df, get_id_from_acronym(df, key), [] + ) + elif isinstance(key, int): + list_parent_ids = tree_traverse_child2parent(df, key, []) + + if isinstance(key, str): # if input is acronym convert IDs to acronyms + parents = [] + for id in list_parent_ids: + parents.append(get_acronym_from_id(df, id)) + elif isinstance(key, int): + parents = list_parent_ids.copy() + + return parents + + +############################################################################## + +############################################################################## +# %% + + +def create_atlas(working_dir, resolution): + ATLAS_NAME = "perens_stereotaxic_mri_mouse" + SPECIES = "Mus musculus" + ATLAS_LINK = "https://www.neuropedia.dk/resource/multimodal-3d-mouse-brain-atlas-framework-with-the-skull-derived-coordinate-system/" + CITATION = "Perens et al. 2023, https://doi.org/10.1007/s12021-023-09623-9" + ORIENTATION = "ial" + ROOT_ID = 997 + ATLAS_FILE_URL = "https://www.neuropedia.dk/wp-content/uploads/Multimodal_mouse_brain_atlas_files.7z" + + # Temporary folder for download: + download_dir_path = working_dir / "downloads" + download_dir_path.mkdir(exist_ok=True) + atlas_files_dir = download_dir_path / "atlas_files" + + ## Download atlas_file + utils.check_internet_connection() + + import urllib.request + + destination_path = download_dir_path / "atlas_download.7z" + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0", + "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8", + "Accept-Language": "en-GB,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br, zstd", + "DNT": "1", + "Sec-GPC": "1", + "Host":"www.neuropedia.dk", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "TE": "trailers", + "Priority":"u=0, i" + } + if not os.path.isdir(atlas_files_dir / 'Multimodal_mouse_brain_atlas_files'): + req = urllib.request.Request(ATLAS_FILE_URL, headers=headers) + with urllib.request.urlopen(req) as response, open(destination_path, 'wb') as out_file: + data = response.read() # a `bytes` object + out_file.write(data) + with py7zr.SevenZipFile(destination_path, mode='r') as z: + z.extractall(path=atlas_files_dir) + destination_path.unlink() + + structures_file = ( + atlas_files_dir + / 'Multimodal_mouse_brain_atlas_files' + / "Hierarchy_tree" + / "Annotation_info.csv" + ) + annotations_file = ( + atlas_files_dir + / 'Multimodal_mouse_brain_atlas_files' + / "MRI_space_oriented" + / "mri_ano.nii.gz" + ) + reference_file = ( + atlas_files_dir + / 'Multimodal_mouse_brain_atlas_files' + / "MRI_space_oriented" + / "mri_temp.nii.gz" + ) + + annotated_volume = sitk.GetArrayFromImage( + sitk.ReadImage(str(annotations_file)) + ) + template_volume = sitk.GetArrayFromImage( + sitk.ReadImage(str(reference_file)) + ) + + print("Download completed...") + + # ------------------------ # + # STRUCTURES HIERARCHY # + # ------------------------ # + + # Parse region names & hierarchy + # ############################## + df = pd.read_csv(structures_file) + + # Make region hierarchy and gather colors to one list + parents = [] + rgb = [] + for index, row in df.iterrows(): + temp_id = row["id"] + temp_parents = get_all_parents(df, temp_id) + parents.append(temp_parents[::-1]) + + temp_rgb = [row["red"], row["green"], row["blue"]] + rgb.append(temp_rgb) + + df = df.drop(columns=["parent_id", "red", "green", "blue"]) + df = df.assign(structure_id_path=parents) + df = df.assign(rgb_triplet=rgb) + df.loc[0, "structure_id_path"] = [997] + + structures = df.to_dict("records") + + for structure in structures: + # root doesn't have a parent + if structure["id"] != 997: + structure["structure_id_path"].append(structure["id"]) + + # save regions list json: + with open(download_dir_path / "structures.json", "w") as f: + json.dump(structures, f) + + # Create meshes: + print(f"Saving atlas data at {download_dir_path}") + meshes_dir_path = download_dir_path / "meshes" + meshes_dir_path.mkdir(exist_ok=True) + + tree = get_structures_tree(structures) + + labels = np.unique(annotated_volume).astype(np.int32) + for key, node in tree.nodes.items(): + if key in labels: + is_label = True + else: + is_label = False + + node.data = Region(is_label) + + # Mesh creation + closing_n_iters = 2 # not used for this atlas + decimate_fraction = 0.2 # not used for this atlas + + smooth = False + start = time.time() + if PARALLEL: + pool = mp.Pool(mp.cpu_count() - 2) + + try: + pool.map( + create_region_mesh, + [ + ( + meshes_dir_path, + node, + tree, + labels, + annotated_volume, + ROOT_ID, + closing_n_iters, + decimate_fraction, + smooth, + ) + for node in tree.nodes.values() + ], + ) + except mp.pool.MaybeEncodingError: + # error with returning results from pool.map but we don't care + pass + else: + for node in track( + tree.nodes.values(), + total=tree.size(), + description="Creating meshes", + ): + create_region_mesh( + ( + meshes_dir_path, + node, + tree, + labels, + annotated_volume, + ROOT_ID, + closing_n_iters, + decimate_fraction, + smooth, + ) + ) + + print( + "Finished mesh extraction in: ", + round((time.time() - start) / 60, 2), + " minutes", + ) + + # Create meshes dict + meshes_dict = dict() + structures_with_mesh = [] + for s in structures: + # Check if a mesh was created + mesh_path = meshes_dir_path / f'{s["id"]}.obj' + if not mesh_path.exists(): + print(f"No mesh file exists for: {s}, ignoring it") + continue + else: + # Check that the mesh actually exists (i.e. not empty) + if mesh_path.stat().st_size < 512: + print(f"obj file for {s} is too small, ignoring it.") + continue + + structures_with_mesh.append(s) + meshes_dict[s["id"]] = mesh_path + + print( + f"In the end, {len(structures_with_mesh)} " + "structures with mesh are kept" + ) + + # ----------- # + # WRAP UP # + # ----------- # + + # Wrap up, compress, and remove file: + print("Finalising atlas") + output_filename = wrapup_atlas_from_data( + atlas_name=ATLAS_NAME, + atlas_minor_version=__version__, + citation=CITATION, + atlas_link=ATLAS_LINK, + species=SPECIES, + resolution=(resolution,) * 3, + orientation=ORIENTATION, + root_id=ROOT_ID, + reference_stack=template_volume, + annotation_stack=annotated_volume, + structures_list=structures_with_mesh, + meshes_dict=meshes_dict, + working_dir=working_dir, + hemispheres_stack=None, + cleanup_files=False, + compress=True, + scale_meshes=True, + ) + + return output_filename + + +if __name__ == "__main__": + resolution = 25 # some resolution, in microns + + # Generated atlas path: + bg_root_dir = Path.home() / "brainglobe_workingdir" / "perens_mri_mouse" + bg_root_dir.mkdir(exist_ok=True, parents=True) + create_atlas(bg_root_dir, resolution) From 426e0331008c45fca9c677bd105f078d582def96 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Fri, 23 Aug 2024 18:55:41 +0200 Subject: [PATCH 02/25] update requirement for perens_mri --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index ec0e5083..6b7bf7b5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,7 @@ allenmouse = ["allensdk"] allenmouse_barrels = [ "allensdk", "voxcell"] +perens_stereotaxic_mri_mouse = ["py7zr"] atlasgen = [ "loguru", From 5866489c91acd5b0391bf40c2fb1a57cf9c8e210 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:08:07 +0200 Subject: [PATCH 03/25] remove hardcoded path --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index e310c3cd..5f818907 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -1,6 +1,3 @@ -import sys -sys.path.append(r"/home/harryc/github/brainglobe-atlasapi/") - __version__ = "3" import os From 7f1d9eb948c4e76d7ed39145ceb34a3166258818 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:08:28 +0000 Subject: [PATCH 04/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../perens_stereotaxic_mri_mouse.py | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 5f818907..6060ca2d 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -1,17 +1,16 @@ - __version__ = "3" -import os import json import multiprocessing as mp -import tarfile +import os import time from pathlib import Path -import requests + import numpy as np import pandas as pd +import py7zr import SimpleITK as sitk from rich.progress import track -import py7zr + from brainglobe_atlasapi import utils from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, @@ -167,7 +166,7 @@ def create_atlas(working_dir, resolution): "Accept-Encoding": "gzip, deflate, br, zstd", "DNT": "1", "Sec-GPC": "1", - "Host":"www.neuropedia.dk", + "Host": "www.neuropedia.dk", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", "Sec-Fetch-Dest": "document", @@ -175,32 +174,37 @@ def create_atlas(working_dir, resolution): "Sec-Fetch-Site": "none", "Sec-Fetch-User": "?1", "TE": "trailers", - "Priority":"u=0, i" + "Priority": "u=0, i", } - if not os.path.isdir(atlas_files_dir / 'Multimodal_mouse_brain_atlas_files'): + if not os.path.isdir( + atlas_files_dir / "Multimodal_mouse_brain_atlas_files" + ): req = urllib.request.Request(ATLAS_FILE_URL, headers=headers) - with urllib.request.urlopen(req) as response, open(destination_path, 'wb') as out_file: + with ( + urllib.request.urlopen(req) as response, + open(destination_path, "wb") as out_file, + ): data = response.read() # a `bytes` object out_file.write(data) - with py7zr.SevenZipFile(destination_path, mode='r') as z: + with py7zr.SevenZipFile(destination_path, mode="r") as z: z.extractall(path=atlas_files_dir) destination_path.unlink() structures_file = ( atlas_files_dir - / 'Multimodal_mouse_brain_atlas_files' + / "Multimodal_mouse_brain_atlas_files" / "Hierarchy_tree" / "Annotation_info.csv" ) annotations_file = ( atlas_files_dir - / 'Multimodal_mouse_brain_atlas_files' + / "Multimodal_mouse_brain_atlas_files" / "MRI_space_oriented" / "mri_ano.nii.gz" ) reference_file = ( atlas_files_dir - / 'Multimodal_mouse_brain_atlas_files' + / "Multimodal_mouse_brain_atlas_files" / "MRI_space_oriented" / "mri_temp.nii.gz" ) From d479291564b9df3dbbed7a165dcb050cc80a060a Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:18:29 +0200 Subject: [PATCH 05/25] fix formatting to pass checks --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 6060ca2d..7451e120 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -161,7 +161,16 @@ def create_atlas(working_dir, resolution): destination_path = download_dir_path / "atlas_download.7z" headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0", - "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/png,image/svg+xml,*/*;q=0.8", + "Accept": ( + "text/html," + "application/xhtml+xml," + "application/xml;q=0.9," + "image/avif," + "image/webp," + "image/png," + "image/svg+xml," + "*/*;q=0.8" + ), "Accept-Language": "en-GB,en;q=0.5", "Accept-Encoding": "gzip, deflate, br, zstd", "DNT": "1", From 2128e25ec74db4e6a83496b4dac93bfd077a129b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:18:36 +0000 Subject: [PATCH 06/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../perens_stereotaxic_mri_mouse.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 7451e120..855278af 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -162,15 +162,15 @@ def create_atlas(working_dir, resolution): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0", "Accept": ( - "text/html," - "application/xhtml+xml," - "application/xml;q=0.9," - "image/avif," - "image/webp," - "image/png," - "image/svg+xml," - "*/*;q=0.8" - ), + "text/html," + "application/xhtml+xml," + "application/xml;q=0.9," + "image/avif," + "image/webp," + "image/png," + "image/svg+xml," + "*/*;q=0.8" + ), "Accept-Language": "en-GB,en;q=0.5", "Accept-Encoding": "gzip, deflate, br, zstd", "DNT": "1", From d8215edc24bd762ce84cc0896838fa49a228ac2d Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Fri, 23 Aug 2024 19:20:50 +0200 Subject: [PATCH 07/25] fix linting issue --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 855278af..ee772ec3 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -160,7 +160,14 @@ def create_atlas(working_dir, resolution): destination_path = download_dir_path / "atlas_download.7z" headers = { - "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0", + "User-Agent": ( + "Mozilla/5.0 " + "(Windows NT 10.0; " + "Win64; x64; " + "rv:129.0) " + "Gecko/20100101 " + "Firefox/129.0" + ), "Accept": ( "text/html," "application/xhtml+xml," From 5c5f8868ee9cc20838a99b09c58b1c0048b24969 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 17:20:59 +0000 Subject: [PATCH 08/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index ee772ec3..1c02c806 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -161,13 +161,13 @@ def create_atlas(working_dir, resolution): destination_path = download_dir_path / "atlas_download.7z" headers = { "User-Agent": ( - "Mozilla/5.0 " - "(Windows NT 10.0; " - "Win64; x64; " - "rv:129.0) " - "Gecko/20100101 " - "Firefox/129.0" - ), + "Mozilla/5.0 " + "(Windows NT 10.0; " + "Win64; x64; " + "rv:129.0) " + "Gecko/20100101 " + "Firefox/129.0" + ), "Accept": ( "text/html," "application/xhtml+xml," From af314843b59b2be6724bdda2cfe36a4dc140e27d Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Tue, 27 Aug 2024 22:38:40 +0200 Subject: [PATCH 09/25] Update brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py Co-authored-by: Adam Tyson --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 1c02c806..84d4493a 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -1,4 +1,4 @@ -__version__ = "3" +__version__ = "0" import json import multiprocessing as mp import os From 705b4f73c0441d32d7453747aa9753adcdb3f3a1 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:01:14 +0200 Subject: [PATCH 10/25] remove sitk in favour of brainglobe utils --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 84d4493a..4291d71b 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -8,7 +8,6 @@ import numpy as np import pandas as pd import py7zr -import SimpleITK as sitk from rich.progress import track from brainglobe_atlasapi import utils @@ -225,12 +224,8 @@ def create_atlas(working_dir, resolution): / "mri_temp.nii.gz" ) - annotated_volume = sitk.GetArrayFromImage( - sitk.ReadImage(str(annotations_file)) - ) - template_volume = sitk.GetArrayFromImage( - sitk.ReadImage(str(reference_file)) - ) + annotated_volume = brainglobe_utils.IO.image.load_any(annotations_file) + template_volume = brainglobe_utils.IO.image.load_any(reference_file) print("Download completed...") From fac995d74916eca6a24f4f0417fc0e5f5da54296 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:24:42 +0200 Subject: [PATCH 11/25] move download and extraction logic into a seperate function --- .../perens_stereotaxic_mri_mouse.py | 97 ++++++++++--------- 1 file changed, 51 insertions(+), 46 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 4291d71b..fd495b33 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -11,6 +11,7 @@ from rich.progress import track from brainglobe_atlasapi import utils +from brainglobe_utils.IO.image import load_any from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, create_region_mesh, @@ -20,7 +21,51 @@ PARALLEL = True # disable parallel mesh extraction for easier debugging - +HEADERS = { + "User-Agent": ( + "Mozilla/5.0 " + "(Windows NT 10.0; " + "Win64; x64; " + "rv:129.0) " + "Gecko/20100101 " + "Firefox/129.0" + ), + "Accept": ( + "text/html," + "application/xhtml+xml," + "application/xml;q=0.9," + "image/avif," + "image/webp," + "image/png," + "image/svg+xml," + "*/*;q=0.8" + ), + "Accept-Language": "en-GB,en;q=0.5", + "Accept-Encoding": "gzip, deflate, br, zstd", + "DNT": "1", + "Sec-GPC": "1", + "Host": "www.neuropedia.dk", + "Connection": "keep-alive", + "Upgrade-Insecure-Requests": "1", + "Sec-Fetch-Dest": "document", + "Sec-Fetch-Mode": "navigate", + "Sec-Fetch-Site": "none", + "Sec-Fetch-User": "?1", + "TE": "trailers", + "Priority": "u=0, i", +} +def download_and_extract_files(ATLAS_FILE_URL, destination_path): + """This is needed to get the brainglobe data from their server, + and bypass cloudflare which is only allowing browser access""" + req = urllib.request.Request(ATLAS_FILE_URL, headers=HEADERS) + with ( + urllib.request.urlopen(req) as response, + open(destination_path, "wb") as out_file, + ): + data = response.read() # a `bytes` object + out_file.write(data) + with py7zr.SevenZipFile(destination_path, mode="r") as z: + z.extractall(path=atlas_files_dir) ### Additional functions ##################################################### @@ -135,7 +180,7 @@ def get_all_parents(df, key): ############################################################################## ############################################################################## -# %% + def create_atlas(working_dir, resolution): @@ -158,51 +203,11 @@ def create_atlas(working_dir, resolution): import urllib.request destination_path = download_dir_path / "atlas_download.7z" - headers = { - "User-Agent": ( - "Mozilla/5.0 " - "(Windows NT 10.0; " - "Win64; x64; " - "rv:129.0) " - "Gecko/20100101 " - "Firefox/129.0" - ), - "Accept": ( - "text/html," - "application/xhtml+xml," - "application/xml;q=0.9," - "image/avif," - "image/webp," - "image/png," - "image/svg+xml," - "*/*;q=0.8" - ), - "Accept-Language": "en-GB,en;q=0.5", - "Accept-Encoding": "gzip, deflate, br, zstd", - "DNT": "1", - "Sec-GPC": "1", - "Host": "www.neuropedia.dk", - "Connection": "keep-alive", - "Upgrade-Insecure-Requests": "1", - "Sec-Fetch-Dest": "document", - "Sec-Fetch-Mode": "navigate", - "Sec-Fetch-Site": "none", - "Sec-Fetch-User": "?1", - "TE": "trailers", - "Priority": "u=0, i", - } + if not os.path.isdir( atlas_files_dir / "Multimodal_mouse_brain_atlas_files" ): - req = urllib.request.Request(ATLAS_FILE_URL, headers=headers) - with ( - urllib.request.urlopen(req) as response, - open(destination_path, "wb") as out_file, - ): - data = response.read() # a `bytes` object - out_file.write(data) - with py7zr.SevenZipFile(destination_path, mode="r") as z: - z.extractall(path=atlas_files_dir) + download_and_extract_files(ATLAS_FILE_URL, destination_path) destination_path.unlink() structures_file = ( @@ -224,8 +229,8 @@ def create_atlas(working_dir, resolution): / "mri_temp.nii.gz" ) - annotated_volume = brainglobe_utils.IO.image.load_any(annotations_file) - template_volume = brainglobe_utils.IO.image.load_any(reference_file) + annotated_volume = load_any(annotations_file) + template_volume = load_any(reference_file) print("Download completed...") From 71b9f5173242baa7c32cc8ad29b118b6938eef8a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:25:11 +0000 Subject: [PATCH 12/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index fd495b33..729b6bf2 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -8,10 +8,10 @@ import numpy as np import pandas as pd import py7zr +from brainglobe_utils.IO.image import load_any from rich.progress import track from brainglobe_atlasapi import utils -from brainglobe_utils.IO.image import load_any from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, create_region_mesh, @@ -54,6 +54,8 @@ "TE": "trailers", "Priority": "u=0, i", } + + def download_and_extract_files(ATLAS_FILE_URL, destination_path): """This is needed to get the brainglobe data from their server, and bypass cloudflare which is only allowing browser access""" @@ -66,6 +68,8 @@ def download_and_extract_files(ATLAS_FILE_URL, destination_path): out_file.write(data) with py7zr.SevenZipFile(destination_path, mode="r") as z: z.extractall(path=atlas_files_dir) + + ### Additional functions ##################################################### @@ -182,7 +186,6 @@ def get_all_parents(df, key): ############################################################################## - def create_atlas(working_dir, resolution): ATLAS_NAME = "perens_stereotaxic_mri_mouse" SPECIES = "Mus musculus" @@ -200,8 +203,6 @@ def create_atlas(working_dir, resolution): ## Download atlas_file utils.check_internet_connection() - import urllib.request - destination_path = download_dir_path / "atlas_download.7z" if not os.path.isdir( From dff50a7a2d02c0a4aa0e8178196bf08b8a257c18 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Wed, 28 Aug 2024 17:37:39 +0200 Subject: [PATCH 13/25] replace functions that could instead use bg atlas functionality --- .../perens_stereotaxic_mri_mouse.py | 102 ++---------------- 1 file changed, 11 insertions(+), 91 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 729b6bf2..9c6ed742 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -8,17 +8,18 @@ import numpy as np import pandas as pd import py7zr -from brainglobe_utils.IO.image import load_any from rich.progress import track from brainglobe_atlasapi import utils +from brainglobe_utils.IO.image import load_any from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, create_region_mesh, ) from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree - +from brainglobe_atlasapi import BrainGlobeAtlas +atlas = BrainGlobeAtlas("allen_mouse_25um") PARALLEL = True # disable parallel mesh extraction for easier debugging HEADERS = { @@ -54,8 +55,6 @@ "TE": "trailers", "Priority": "u=0, i", } - - def download_and_extract_files(ATLAS_FILE_URL, destination_path): """This is needed to get the brainglobe data from their server, and bypass cloudflare which is only allowing browser access""" @@ -68,75 +67,8 @@ def download_and_extract_files(ATLAS_FILE_URL, destination_path): out_file.write(data) with py7zr.SevenZipFile(destination_path, mode="r") as z: z.extractall(path=atlas_files_dir) - - ### Additional functions ##################################################### - -############################################################################## -def get_id_from_acronym(df, acronym): - """ - Get Allen's brain atlas ID from brain region acronym(s) - - Call: - get_id_from_acronym(df, acronym) - - Args: - df (pandas dataframe): - atlas table file [see atlas.load_table()] - acronym (string or list of strings): brain region acronym(s) - - Returns: - ID (int or list of ints): - brain region ID(s) corresponding to input acronym(s) - """ - - # create as list if necessary - if not isinstance(acronym, list): - acronym = [acronym] - - if len(acronym) > 1: - ID_list = [] - for acro in acronym: - ID = df["id"][df["acronym"] == acro].item() - ID_list.append(ID) - return ID_list - else: - return df["id"][df["acronym"] == acronym[0]].item() - - # return df['id'][df['acronym'] == acronym].item() # OLD VERSION - - -def get_acronym_from_id(df, ID): - """ - Get Allen's brain atlas acronym from brain region ID(s) - - Call: - get_acronym_from_ID(df, acronym) - - Args: - df (pandas dataframe): atlas table dataframe [see atlas.load_table()] - ID (int or list of int): brain region ID(s) - - Returns: - acronym (string or list of strings): - brain region acronym(s) corresponding to input ID(s) - """ - - # create as list if necessary - if not isinstance(ID, list): - ID = [ID] - - if len(ID) > 1: - acronym_list = [] - for id in ID: - acronym = df["acronym"][df["id"] == id].item() - acronym_list.append(acronym) - return acronym_list - else: - return df["acronym"][df["id"] == ID[0]].item() - - def tree_traverse_child2parent(df, child_id, ids): parent = df["parent_id"][df["id"] == child_id].item() @@ -149,7 +81,7 @@ def tree_traverse_child2parent(df, child_id, ids): return ids -def get_all_parents(df, key): +def get_all_parents(atlas, key): """ Get all parent IDs/acronyms in Allen's brain atlas hierarchical structure' @@ -157,28 +89,14 @@ def get_all_parents(df, key): get_all_children(df, key) Args: - df (pandas dataframe) : atlas table dataframe [see atlas.load_table()] key (int/string) : atlas region ID/acronym Returns: parents (list) : brain region acronym corresponding to input ID """ - - if isinstance(key, str): # if input is acronym convert to ID - list_parent_ids = tree_traverse_child2parent( - df, get_id_from_acronym(df, key), [] - ) - elif isinstance(key, int): - list_parent_ids = tree_traverse_child2parent(df, key, []) - - if isinstance(key, str): # if input is acronym convert IDs to acronyms - parents = [] - for id in list_parent_ids: - parents.append(get_acronym_from_id(df, id)) - elif isinstance(key, int): - parents = list_parent_ids.copy() - - return parents + parents = atlas.get_structure_ancestors(temp_id) + parent_ids = [atlas.structures.acronym_to_id_map[p] for p in parents] + return parent_ids ############################################################################## @@ -186,6 +104,7 @@ def get_all_parents(df, key): ############################################################################## + def create_atlas(working_dir, resolution): ATLAS_NAME = "perens_stereotaxic_mri_mouse" SPECIES = "Mus musculus" @@ -203,6 +122,8 @@ def create_atlas(working_dir, resolution): ## Download atlas_file utils.check_internet_connection() + import urllib.request + destination_path = download_dir_path / "atlas_download.7z" if not os.path.isdir( @@ -248,9 +169,8 @@ def create_atlas(working_dir, resolution): rgb = [] for index, row in df.iterrows(): temp_id = row["id"] - temp_parents = get_all_parents(df, temp_id) + temp_parents = get_all_parents(atlas, temp_id) parents.append(temp_parents[::-1]) - temp_rgb = [row["red"], row["green"], row["blue"]] rgb.append(temp_rgb) From 112a4b91520c398bba93d8b4aa0bd3539fbb62c7 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 15:39:16 +0000 Subject: [PATCH 14/25] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 9c6ed742..921a43a2 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -8,17 +8,17 @@ import numpy as np import pandas as pd import py7zr +from brainglobe_utils.IO.image import load_any from rich.progress import track -from brainglobe_atlasapi import utils -from brainglobe_utils.IO.image import load_any +from brainglobe_atlasapi import BrainGlobeAtlas, utils from brainglobe_atlasapi.atlas_generation.mesh_utils import ( Region, create_region_mesh, ) from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree -from brainglobe_atlasapi import BrainGlobeAtlas + atlas = BrainGlobeAtlas("allen_mouse_25um") PARALLEL = True # disable parallel mesh extraction for easier debugging @@ -55,6 +55,8 @@ "TE": "trailers", "Priority": "u=0, i", } + + def download_and_extract_files(ATLAS_FILE_URL, destination_path): """This is needed to get the brainglobe data from their server, and bypass cloudflare which is only allowing browser access""" @@ -67,8 +69,11 @@ def download_and_extract_files(ATLAS_FILE_URL, destination_path): out_file.write(data) with py7zr.SevenZipFile(destination_path, mode="r") as z: z.extractall(path=atlas_files_dir) + + ### Additional functions ##################################################### + def tree_traverse_child2parent(df, child_id, ids): parent = df["parent_id"][df["id"] == child_id].item() @@ -104,7 +109,6 @@ def get_all_parents(atlas, key): ############################################################################## - def create_atlas(working_dir, resolution): ATLAS_NAME = "perens_stereotaxic_mri_mouse" SPECIES = "Mus musculus" @@ -122,8 +126,6 @@ def create_atlas(working_dir, resolution): ## Download atlas_file utils.check_internet_connection() - import urllib.request - destination_path = download_dir_path / "atlas_download.7z" if not os.path.isdir( From f129f23efd1cd6ee3cd2cf6582b051f3a85742ff Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:36:42 +0200 Subject: [PATCH 15/25] Add stereotaxic atlas to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 449a7245..8271d5cf 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,8 @@ A number of atlases are in development, but those available currently are: | [Blind Mexican Cavefish Brain Atlas](https://doi.org/10.7554/eLife.80777) | 2 micron | 1 year | IHC | [BlueBrain Barrel Cortex Atlas](https://doi.org/10.1162/imag_a_00209) | 10 and 25 micron | P56 | STPT | [UNAM Axolotl Brain Atlas](https://doi.org/10.1038/s41598-021-89357-3) | 40 micron | ~ 3 months post hatching | MRI +| [Gubra's stereotaxic MRI mouse brain atlas](https://doi.org/10.1007/s12021-023-09623-9) | 25 micron | 10-week-old | MRI + ## Installation From fead940503dbc161ba46230a4400953e7ebf85e3 Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:01:04 +0200 Subject: [PATCH 16/25] Update brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py Co-authored-by: Alessandro Felder --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 921a43a2..3d561c4b 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -19,7 +19,8 @@ from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree -atlas = BrainGlobeAtlas("allen_mouse_25um") +# The Perens atlas re-uses information from the Allen atlas, so it's useful to have an instance of the Allen atlas around +allen_atlas = BrainGlobeAtlas("allen_mouse_25um") PARALLEL = True # disable parallel mesh extraction for easier debugging HEADERS = { From 0efb276fb25f2690257d6cd897c59d4f68bdbf3c Mon Sep 17 00:00:00 2001 From: Harry Carey <38996929+PolarBean@users.noreply.github.com> Date: Thu, 29 Aug 2024 15:02:03 +0200 Subject: [PATCH 17/25] Update brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py Co-authored-by: Alessandro Felder --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 3d561c4b..4ff4f88f 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -59,7 +59,7 @@ def download_and_extract_files(ATLAS_FILE_URL, destination_path): - """This is needed to get the brainglobe data from their server, + """This is needed to get the source data from their server, and bypass cloudflare which is only allowing browser access""" req = urllib.request.Request(ATLAS_FILE_URL, headers=HEADERS) with ( From b05021f3c38ab705160e1961b77e4a036ffbbadd Mon Sep 17 00:00:00 2001 From: polarbean Date: Wed, 11 Sep 2024 17:48:00 +0200 Subject: [PATCH 18/25] update to follow new template --- .../perens_stereotaxic_mri_mouse.py | 233 +++++++++--------- 1 file changed, 123 insertions(+), 110 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 4ff4f88f..ba99024a 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -1,12 +1,10 @@ -__version__ = "0" -import json import multiprocessing as mp import os import time +import urllib.request from pathlib import Path import numpy as np -import pandas as pd import py7zr from brainglobe_utils.IO.image import load_any from rich.progress import track @@ -19,7 +17,8 @@ from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data from brainglobe_atlasapi.structure_tree_util import get_structures_tree -# The Perens atlas re-uses information from the Allen atlas, so it's useful to have an instance of the Allen atlas around +# The Perens atlas re-uses information from the Allen atlas, so it's useful to +# have an instance of the Allen atlas around allen_atlas = BrainGlobeAtlas("allen_mouse_25um") PARALLEL = True # disable parallel mesh extraction for easier debugging @@ -58,72 +57,69 @@ } -def download_and_extract_files(ATLAS_FILE_URL, destination_path): - """This is needed to get the source data from their server, - and bypass cloudflare which is only allowing browser access""" - req = urllib.request.Request(ATLAS_FILE_URL, headers=HEADERS) - with ( - urllib.request.urlopen(req) as response, - open(destination_path, "wb") as out_file, - ): - data = response.read() # a `bytes` object - out_file.write(data) - with py7zr.SevenZipFile(destination_path, mode="r") as z: - z.extractall(path=atlas_files_dir) - - -### Additional functions ##################################################### - - -def tree_traverse_child2parent(df, child_id, ids): - parent = df["parent_id"][df["id"] == child_id].item() +# Copy-paste this script into a new file and fill in the functions to package +# your own atlas. - if not np.isnan(parent): - id = df["id"][df["id"] == parent].item() - ids.append(id) - tree_traverse_child2parent(df, parent, ids) - return ids - else: - return ids +### Metadata ### +# The minor version of the atlas in the brainglobe_atlasapi, this is internal, +# if this is the first time this atlas has been added the value should be 0 +# (minor version is the first number after the decimal point, ie the minor +# version of 1.2 is 2) +__version__ = 0 -def get_all_parents(atlas, key): - """ - Get all parent IDs/acronyms in Allen's brain atlas hierarchical structure' +# The expected format is FirstAuthor_SpeciesCommonName, e.g. kleven_rat, or +# Institution_SpeciesCommonName, e.g. allen_mouse. +ATLAS_NAME = "perens_stereotaxic_mri_mouse" - Call: - get_all_children(df, key) +# DOI of the most relevant citable document +CITATION = "Perens et al. 2023, https://doi.org/10.1007/s12021-023-09623-9" - Args: - key (int/string) : atlas region ID/acronym +# The scientific name of the species, ie; Rattus norvegicus +SPECIES = "Mus musculus" - Returns: - parents (list) : brain region acronym corresponding to input ID - """ - parents = atlas.get_structure_ancestors(temp_id) - parent_ids = [atlas.structures.acronym_to_id_map[p] for p in parents] - return parent_ids +# The URL for the data files +ATLAS_LINK = "https://www.neuropedia.dk/resource/multimodal-3d-mouse-brain-atlas-framework-with-the-skull-derived-coordinate-system/" +ATLAS_FILE_URL = "https://www.neuropedia.dk/wp-content/uploads/Multimodal_mouse_brain_atlas_files.7z" +# The orientation of the **original** atlas data, in BrainGlobe convention: +# https://brainglobe.info/documentation/setting-up/image-definition.html#orientation +ORIENTATION = "ial" +# The id of the highest level of the atlas. This is commonly called root or +# brain. Include some information on what to do if your atlas is not +# hierarchical +ROOT_ID = 997 -############################################################################## +# The resolution of your volume in microns. Details on how to format this +# parameter for non isotropic datasets or datasets with multiple resolutions. +RESOLUTION = 25 -############################################################################## +BG_ROOT_DIR = Path.home() / "brainglobe_workingdir" / ATLAS_NAME -def create_atlas(working_dir, resolution): - ATLAS_NAME = "perens_stereotaxic_mri_mouse" - SPECIES = "Mus musculus" - ATLAS_LINK = "https://www.neuropedia.dk/resource/multimodal-3d-mouse-brain-atlas-framework-with-the-skull-derived-coordinate-system/" - CITATION = "Perens et al. 2023, https://doi.org/10.1007/s12021-023-09623-9" - ORIENTATION = "ial" - ROOT_ID = 997 - ATLAS_FILE_URL = "https://www.neuropedia.dk/wp-content/uploads/Multimodal_mouse_brain_atlas_files.7z" +def download_resources(): + """ + Download the necessary resources for the atlas. - # Temporary folder for download: - download_dir_path = working_dir / "downloads" + If possible, please use the Pooch library to retrieve any resources. + """ + download_dir_path = BG_ROOT_DIR / "downloads" download_dir_path.mkdir(exist_ok=True) atlas_files_dir = download_dir_path / "atlas_files" + def download_and_extract_files(ATLAS_FILE_URL, destination_path): + """This is needed to get the source data from their server, + and bypass cloudflare which is only allowing browser access""" + req = urllib.request.Request(ATLAS_FILE_URL, headers=HEADERS) + with ( + urllib.request.urlopen(req) as response, + open(destination_path, "wb") as out_file, + ): + data = response.read() # a `bytes` object + out_file.write(data) + with py7zr.SevenZipFile(destination_path, mode="r") as z: + z.extractall(path=atlas_files_dir) + ## Download atlas_file utils.check_internet_connection() @@ -135,12 +131,18 @@ def create_atlas(working_dir, resolution): download_and_extract_files(ATLAS_FILE_URL, destination_path) destination_path.unlink() - structures_file = ( - atlas_files_dir - / "Multimodal_mouse_brain_atlas_files" - / "Hierarchy_tree" - / "Annotation_info.csv" - ) + +def retrieve_reference_and_annotation(): + """ + Retrieve the desired reference and annotation as two numpy arrays. + + Returns: + tuple: A tuple containing two numpy arrays. The first array is the + reference volume, and the second array is the annotation volume. + """ + download_dir_path = BG_ROOT_DIR / "downloads" + atlas_files_dir = download_dir_path / "atlas_files" + annotations_file = ( atlas_files_dir / "Multimodal_mouse_brain_atlas_files" @@ -157,44 +159,59 @@ def create_atlas(working_dir, resolution): annotated_volume = load_any(annotations_file) template_volume = load_any(reference_file) - print("Download completed...") + return template_volume, annotated_volume - # ------------------------ # - # STRUCTURES HIERARCHY # - # ------------------------ # - # Parse region names & hierarchy - # ############################## - df = pd.read_csv(structures_file) +def retrieve_hemisphere_map(): + """ + Retrieve a hemisphere map for the atlas. - # Make region hierarchy and gather colors to one list - parents = [] - rgb = [] - for index, row in df.iterrows(): - temp_id = row["id"] - temp_parents = get_all_parents(atlas, temp_id) - parents.append(temp_parents[::-1]) - temp_rgb = [row["red"], row["green"], row["blue"]] - rgb.append(temp_rgb) + If your atlas is asymmetrical, you may want to use a hemisphere map. + This is an array in the same shape as your template, + with 0's marking the left hemisphere, and 1's marking the right. - df = df.drop(columns=["parent_id", "red", "green", "blue"]) - df = df.assign(structure_id_path=parents) - df = df.assign(rgb_triplet=rgb) - df.loc[0, "structure_id_path"] = [997] + If your atlas is symmetrical, ignore this function. - structures = df.to_dict("records") + Returns: + numpy.array or None: A numpy array representing the hemisphere map, + or None if the atlas is symmetrical. + """ + return None - for structure in structures: - # root doesn't have a parent - if structure["id"] != 997: - structure["structure_id_path"].append(structure["id"]) - # save regions list json: - with open(download_dir_path / "structures.json", "w") as f: - json.dump(structures, f) +def retrieve_structure_information(): + """ + Retrieve the structures tree and meshes for the Allen mouse brain atlas. - # Create meshes: - print(f"Saving atlas data at {download_dir_path}") + Returns: + pandas.DataFrame: A DataFrame containing the atlas information. + """ + # Since this atlas inherits from the allen can we not simply get the data + # from the bgapi? + allen_atlas = BrainGlobeAtlas("allen_mouse_25um") + allen_structures = allen_atlas.structures_list + allen_structures = [ + { + "id": i["id"], + "name": i["name"], + "acronym": i["acronym"], + "structure_id_path": i["structure_id_path"], + "rgb_triplet": i["rgb_triplet"], + } + for i in allen_structures + ] + return allen_structures + + +def retrieve_or_construct_meshes(): + """ + This function should return a dictionary of ids and corresponding paths to + mesh files. Some atlases are packaged with mesh files, in these cases we + should use these files. Then this function should download those meshes. + In other cases we need to construct the meshes ourselves. For this we have + helper functions to achieve this. + """ + download_dir_path = BG_ROOT_DIR / "downloads" meshes_dir_path = download_dir_path / "meshes" meshes_dir_path.mkdir(exist_ok=True) @@ -288,39 +305,35 @@ def create_atlas(working_dir, resolution): "structures with mesh are kept" ) - # ----------- # - # WRAP UP # - # ----------- # + return meshes_dict + + +### If the code above this line has been filled correctly, nothing needs to be +### edited below (unless variables need to be passed between the functions). +if __name__ == "__main__": + BG_ROOT_DIR.mkdir(exist_ok=True) + download_resources() + template_volume, annotated_volume = retrieve_reference_and_annotation() + hemispheres_stack = retrieve_hemisphere_map() + structures = retrieve_structure_information() + meshes_dict = retrieve_or_construct_meshes() - # Wrap up, compress, and remove file: - print("Finalising atlas") output_filename = wrapup_atlas_from_data( atlas_name=ATLAS_NAME, atlas_minor_version=__version__, citation=CITATION, atlas_link=ATLAS_LINK, species=SPECIES, - resolution=(resolution,) * 3, + resolution=(RESOLUTION,) * 3, orientation=ORIENTATION, root_id=ROOT_ID, reference_stack=template_volume, annotation_stack=annotated_volume, - structures_list=structures_with_mesh, + structures_list=structures, meshes_dict=meshes_dict, - working_dir=working_dir, + working_dir=BG_ROOT_DIR, hemispheres_stack=None, cleanup_files=False, compress=True, scale_meshes=True, ) - - return output_filename - - -if __name__ == "__main__": - resolution = 25 # some resolution, in microns - - # Generated atlas path: - bg_root_dir = Path.home() / "brainglobe_workingdir" / "perens_mri_mouse" - bg_root_dir.mkdir(exist_ok=True, parents=True) - create_atlas(bg_root_dir, resolution) From dbe44f1ee86620e598178a8b15ca8d196c1053e8 Mon Sep 17 00:00:00 2001 From: polarbean Date: Wed, 11 Sep 2024 18:15:47 +0200 Subject: [PATCH 19/25] removed weird nested function --- .../perens_stereotaxic_mri_mouse.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index ba99024a..d2fcd343 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -107,7 +107,14 @@ def download_resources(): download_dir_path.mkdir(exist_ok=True) atlas_files_dir = download_dir_path / "atlas_files" - def download_and_extract_files(ATLAS_FILE_URL, destination_path): + ## Download atlas_file + utils.check_internet_connection() + + destination_path = download_dir_path / "atlas_download.7z" + + if not os.path.isdir( + atlas_files_dir / "Multimodal_mouse_brain_atlas_files" + ): """This is needed to get the source data from their server, and bypass cloudflare which is only allowing browser access""" req = urllib.request.Request(ATLAS_FILE_URL, headers=HEADERS) @@ -119,16 +126,6 @@ def download_and_extract_files(ATLAS_FILE_URL, destination_path): out_file.write(data) with py7zr.SevenZipFile(destination_path, mode="r") as z: z.extractall(path=atlas_files_dir) - - ## Download atlas_file - utils.check_internet_connection() - - destination_path = download_dir_path / "atlas_download.7z" - - if not os.path.isdir( - atlas_files_dir / "Multimodal_mouse_brain_atlas_files" - ): - download_and_extract_files(ATLAS_FILE_URL, destination_path) destination_path.unlink() From 4b8e085b0ba733bdde16744bae180a41ee020f10 Mon Sep 17 00:00:00 2001 From: Harry Carey Date: Wed, 11 Sep 2024 19:25:48 +0200 Subject: [PATCH 20/25] change template to reference for consistency --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index d2fcd343..d43361a5 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -154,9 +154,9 @@ def retrieve_reference_and_annotation(): ) annotated_volume = load_any(annotations_file) - template_volume = load_any(reference_file) + reference_volume = load_any(reference_file) - return template_volume, annotated_volume + return reference_volume, annotated_volume def retrieve_hemisphere_map(): @@ -310,7 +310,7 @@ def retrieve_or_construct_meshes(): if __name__ == "__main__": BG_ROOT_DIR.mkdir(exist_ok=True) download_resources() - template_volume, annotated_volume = retrieve_reference_and_annotation() + reference_volume, annotated_volume = retrieve_reference_and_annotation() hemispheres_stack = retrieve_hemisphere_map() structures = retrieve_structure_information() meshes_dict = retrieve_or_construct_meshes() @@ -324,7 +324,7 @@ def retrieve_or_construct_meshes(): resolution=(RESOLUTION,) * 3, orientation=ORIENTATION, root_id=ROOT_ID, - reference_stack=template_volume, + reference_stack=reference_volume, annotation_stack=annotated_volume, structures_list=structures, meshes_dict=meshes_dict, From f568d651f36f8e5416d19844ead0e99433a70438 Mon Sep 17 00:00:00 2001 From: polarbean Date: Thu, 12 Sep 2024 17:25:12 +0200 Subject: [PATCH 21/25] update URL to new version, gubra updated this to fix label issue in lsfm --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index d43361a5..956e0d54 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -80,7 +80,7 @@ # The URL for the data files ATLAS_LINK = "https://www.neuropedia.dk/resource/multimodal-3d-mouse-brain-atlas-framework-with-the-skull-derived-coordinate-system/" -ATLAS_FILE_URL = "https://www.neuropedia.dk/wp-content/uploads/Multimodal_mouse_brain_atlas_files.7z" +ATLAS_FILE_URL = "https://www.neuropedia.dk/wp-content/uploads/Multimodal_mouse_brain_atlas_files_v2.7z" # The orientation of the **original** atlas data, in BrainGlobe convention: # https://brainglobe.info/documentation/setting-up/image-definition.html#orientation ORIENTATION = "ial" @@ -111,7 +111,6 @@ def download_resources(): utils.check_internet_connection() destination_path = download_dir_path / "atlas_download.7z" - if not os.path.isdir( atlas_files_dir / "Multimodal_mouse_brain_atlas_files" ): From 1fd2290799672733700f18030ba54e4015bc3004 Mon Sep 17 00:00:00 2001 From: polarbean Date: Thu, 12 Sep 2024 17:28:15 +0200 Subject: [PATCH 22/25] add progress bar --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 956e0d54..9fe4c99d 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -7,7 +7,7 @@ import numpy as np import py7zr from brainglobe_utils.IO.image import load_any -from rich.progress import track +from rich.progress import Progress, track from brainglobe_atlasapi import BrainGlobeAtlas, utils from brainglobe_atlasapi.atlas_generation.mesh_utils import ( @@ -114,15 +114,20 @@ def download_resources(): if not os.path.isdir( atlas_files_dir / "Multimodal_mouse_brain_atlas_files" ): - """This is needed to get the source data from their server, - and bypass cloudflare which is only allowing browser access""" req = urllib.request.Request(ATLAS_FILE_URL, headers=HEADERS) with ( urllib.request.urlopen(req) as response, open(destination_path, "wb") as out_file, ): - data = response.read() # a `bytes` object - out_file.write(data) + total = int(response.headers.get("content-length", 0)) + with Progress() as progress: + task = progress.add_task("[cyan]Downloading...", total=total) + while not progress.finished: + chunk = response.read(1024) + if not chunk: + break + out_file.write(chunk) + progress.update(task, advance=len(chunk)) with py7zr.SevenZipFile(destination_path, mode="r") as z: z.extractall(path=atlas_files_dir) destination_path.unlink() From 178d92e95fa6238745edc52ed76838296dd08b00 Mon Sep 17 00:00:00 2001 From: polarbean Date: Thu, 12 Sep 2024 17:30:34 +0200 Subject: [PATCH 23/25] add more print statements so progress is easier to observe --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 9fe4c99d..c9528c49 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -141,6 +141,7 @@ def retrieve_reference_and_annotation(): tuple: A tuple containing two numpy arrays. The first array is the reference volume, and the second array is the annotation volume. """ + print("loading reference and annotation volume") download_dir_path = BG_ROOT_DIR / "downloads" atlas_files_dir = download_dir_path / "atlas_files" @@ -212,6 +213,8 @@ def retrieve_or_construct_meshes(): In other cases we need to construct the meshes ourselves. For this we have helper functions to achieve this. """ + print("constructing meshes") + download_dir_path = BG_ROOT_DIR / "downloads" meshes_dir_path = download_dir_path / "meshes" meshes_dir_path.mkdir(exist_ok=True) From 239af9a69764cf9c25dedaf53a4a987167454c17 Mon Sep 17 00:00:00 2001 From: polarbean Date: Sun, 15 Sep 2024 11:01:11 +0200 Subject: [PATCH 24/25] added correct orientation --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index c9528c49..1521958c 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -83,7 +83,7 @@ ATLAS_FILE_URL = "https://www.neuropedia.dk/wp-content/uploads/Multimodal_mouse_brain_atlas_files_v2.7z" # The orientation of the **original** atlas data, in BrainGlobe convention: # https://brainglobe.info/documentation/setting-up/image-definition.html#orientation -ORIENTATION = "ial" +ORIENTATION = "als" # The id of the highest level of the atlas. This is commonly called root or # brain. Include some information on what to do if your atlas is not @@ -190,6 +190,7 @@ def retrieve_structure_information(): """ # Since this atlas inherits from the allen can we not simply get the data # from the bgapi? + "determining structures" allen_atlas = BrainGlobeAtlas("allen_mouse_25um") allen_structures = allen_atlas.structures_list allen_structures = [ @@ -321,7 +322,7 @@ def retrieve_or_construct_meshes(): hemispheres_stack = retrieve_hemisphere_map() structures = retrieve_structure_information() meshes_dict = retrieve_or_construct_meshes() - + print("wrapping up atlas") output_filename = wrapup_atlas_from_data( atlas_name=ATLAS_NAME, atlas_minor_version=__version__, From 1250a599afe8d4f432fd8831c25da0be8ba1bb11 Mon Sep 17 00:00:00 2001 From: Harry Carey Date: Fri, 27 Sep 2024 21:48:25 +0200 Subject: [PATCH 25/25] added print statement to structure information function --- .../atlas_scripts/perens_stereotaxic_mri_mouse.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py index 1521958c..37c809d2 100644 --- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py +++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/perens_stereotaxic_mri_mouse.py @@ -190,7 +190,7 @@ def retrieve_structure_information(): """ # Since this atlas inherits from the allen can we not simply get the data # from the bgapi? - "determining structures" + print("determining structures") allen_atlas = BrainGlobeAtlas("allen_mouse_25um") allen_structures = allen_atlas.structures_list allen_structures = [