From f3cf15ec7614bc32711f859851279207065076df Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Tue, 3 Sep 2024 17:38:24 +0100
Subject: [PATCH 01/22] new atlas file added - for cat

---
 .../atlas_generation/atlas_scripts/catlas.py  | 111 ++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
new file mode 100644
index 00000000..2e72e94b
--- /dev/null
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -0,0 +1,111 @@
+from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+
+###Metadata
+__version__ = 1  # The version of the atlas in the brainglobe_atlasapi, this is internal, if this is the first time this atlas has been added the value should be 1
+ATLAS_NAME = None  # The expected format is FirstAuthor_SpeciesCommonName, ie; kleven_rat
+CITATION = None  # DOI of the most relevant citable document
+SPECIES = None  # The scientific name of the species, ie; Rattus norvegicus
+ATLAS_LINK = None  # The URL for the data files
+ORIENTATION = None  # The orientation of the atlas, for more information on how to determine this click here: ........
+ROOT_ID = None  # The id of the highest level of the atlas. This is commonly called root or brain. Include some information on what to do if your atlas is not hierarchical
+RESOLUTION = None  # the resolution of your volume in microns. details on how to format this parameter for non isotropic datasets or datasets with multiple resolutions.
+
+
+def download_resources():
+    """
+    Download the necessary resources for the atlas.
+
+    If possible, please use the Pooch library to retrieve any resources.
+    """
+    pass
+
+
+def retrieve_template_and_reference():
+    """
+    Retrieve the desired template and reference as two numpy arrays.
+
+    Returns:
+        tuple: A tuple containing two numpy arrays. The first array is the template volume, and the second array is the reference volume.
+    """
+    template = None
+    reference = None
+    return template, reference
+
+
+def retrieve_hemisphere_map():
+    """
+    Retrieve a hemisphere map for the atlas.
+
+    If your atlas is asymmetrical, you may want to use a hemisphere map. This is an array in the same shape as your template,
+    with 0's marking the left hemisphere, and 1's marking the right.
+
+    If your atlas is symmetrical, ignore this function.
+
+    Returns:
+        numpy.array or None: A numpy array representing the hemisphere map, or None if the atlas is symmetrical.
+    """
+    return None
+
+
+def retrieve_structure_information():
+    """
+    This function should return a pandas DataFrame with information about your atlas.
+
+    The DataFrame should be in the following format:
+
+    ╭─────────────┬───────────────────────────────────┬─────────┬───────────────────────┬───────────────────╮
+    | id          | name                              | acronym | structure_id_path     | rgb_triplet       |
+    |             |                                   |         |                       |                   |
+    ├─────────────┼───────────────────────────────────┼─────────┼───────────────────────┼───────────────────┤
+    | 997         | root                              | root    | []                    | [255, 255, 255]   |
+    ├─────────────┼───────────────────────────────────┼─────────┼───────────────────────┼───────────────────┤
+    | 8           | Basic cell groups and regions     | grey    | [997]                 | [191, 218, 227]   |
+    ├─────────────┼───────────────────────────────────┼─────────┼───────────────────────┼───────────────────┤
+    | 567         | Cerebrum                          | CH      | [997, 8]              | [176, 240, 255]   |
+    ╰─────────────┴───────────────────────────────────┴─────────┴───────────────────────┴───────────────────╯
+
+    Returns:
+        pandas.DataFrame: A DataFrame containing the atlas information.
+    """
+    return None
+
+
+def retrieve_or_construct_meshes():
+    """
+    This function should return a dictionary of ids and corresponding paths to mesh files.
+    Some atlases are packaged with mesh files, in these cases we should use these files.
+    Then this function should download those meshes. In other cases we need to construct
+    the meshes ourselves. For this we have helper functions to achieve this.
+    """
+    meshes_dict = {}
+    return meshes_dict
+
+
+### If the code above this line has been filled correctly, nothing needs to be edited below (unless variables need to be passed between the functions).
+bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
+bg_root_dir.mkdir(exist_ok=True)
+download_resources()
+template_volume, reference_volume = retrieve_template_and_reference()
+hemispheres_stack = retrieve_hemisphere_map()
+structures = retrieve_structure_information()
+meshes_dict = retrieve_or_construct_meshes()
+
+output_filename = wrapup_atlas_from_data(
+    atlas_name=ATLAS_NAME,
+    atlas_minor_version=__version__,
+    citation=CITATION,
+    atlas_link=ATLAS_LINK,
+    species=SPECIES,
+    resolution=(RESOLUTION,) * 3,
+    orientation=ORIENTATION,
+    root_id=ROOT_ID,
+    reference_stack=template_volume,
+    annotation_stack=annotated_volume,
+    structures_list=structures,
+    meshes_dict=meshes_dict,
+    working_dir=working_dir,
+    hemispheres_stack=None,
+    cleanup_files=False,
+    compress=True,
+    scale_meshes=True,
+)
\ No newline at end of file

From 56cb6f5fba98ca2438fefecd02bb723237812cd1 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Tue, 3 Sep 2024 16:43:04 +0000
Subject: [PATCH 02/22] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 2e72e94b..04bdb183 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -108,4 +108,4 @@ def retrieve_or_construct_meshes():
     cleanup_files=False,
     compress=True,
     scale_meshes=True,
-)
\ No newline at end of file
+)

From 101bfcb557e04cb0ca8b545ba20140cfbaa03afa Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sun, 8 Sep 2024 12:29:24 +0100
Subject: [PATCH 03/22] included relevant metadata - root_id is 999 as labels
 are a non structured list

---
 .../atlas_generation/atlas_scripts/catlas.py  | 35 ++++++++++++-------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 2e72e94b..b2d7880a 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -1,22 +1,30 @@
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+from pathlib import Path
+import pooch 
 
 ###Metadata
-__version__ = 1  # The version of the atlas in the brainglobe_atlasapi, this is internal, if this is the first time this atlas has been added the value should be 1
-ATLAS_NAME = None  # The expected format is FirstAuthor_SpeciesCommonName, ie; kleven_rat
-CITATION = None  # DOI of the most relevant citable document
-SPECIES = None  # The scientific name of the species, ie; Rattus norvegicus
-ATLAS_LINK = None  # The URL for the data files
-ORIENTATION = None  # The orientation of the atlas, for more information on how to determine this click here: ........
-ROOT_ID = None  # The id of the highest level of the atlas. This is commonly called root or brain. Include some information on what to do if your atlas is not hierarchical
-RESOLUTION = None  # the resolution of your volume in microns. details on how to format this parameter for non isotropic datasets or datasets with multiple resolutions.
+__version__ = 1
+ATLAS_NAME = "catlas"
+CITATION = "https://doi.org/10.1002/cne.24271"
+SPECIES = "Felis catus"
+ATLAS_LINK = "https://github.com/CerebralSystemsLab/CATLAS"
+ORIENTATION = "lps"
+ROOT_ID = 999  # Placeholder as no hierarchy is present
+RESOLUTION = 500  # um
+ATLAS_PACKAGER = 'Henry Crosswell'
 
 
 def download_resources():
+
+
     """
     Download the necessary resources for the atlas.
 
     If possible, please use the Pooch library to retrieve any resources.
     """
+
+
+    
     pass
 
 
@@ -25,7 +33,8 @@ def retrieve_template_and_reference():
     Retrieve the desired template and reference as two numpy arrays.
 
     Returns:
-        tuple: A tuple containing two numpy arrays. The first array is the template volume, and the second array is the reference volume.
+        tuple: A tuple containing two numpy arrays. The first array is the template volume, 
+        and the second array is the reference volume.
     """
     template = None
     reference = None
@@ -36,13 +45,15 @@ def retrieve_hemisphere_map():
     """
     Retrieve a hemisphere map for the atlas.
 
-    If your atlas is asymmetrical, you may want to use a hemisphere map. This is an array in the same shape as your template,
+    If your atlas is asymmetrical, you may want to use a hemisphere map. 
+    This is an array in the same shape as your template,
     with 0's marking the left hemisphere, and 1's marking the right.
 
     If your atlas is symmetrical, ignore this function.
 
     Returns:
-        numpy.array or None: A numpy array representing the hemisphere map, or None if the atlas is symmetrical.
+        numpy.array or None: A numpy array representing the hemisphere map, 
+        or None if the atlas is symmetrical.
     """
     return None
 
@@ -108,4 +119,4 @@ def retrieve_or_construct_meshes():
     cleanup_files=False,
     compress=True,
     scale_meshes=True,
-)
\ No newline at end of file
+)

From c18a8dbb7daefb88116c8645503131a3a42dccc8 Mon Sep 17 00:00:00 2001
From: "pre-commit-ci[bot]"
 <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Date: Sun, 8 Sep 2024 11:46:35 +0000
Subject: [PATCH 04/22] [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci
---
 .../atlas_generation/atlas_scripts/catlas.py     | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index b2d7880a..e6bae116 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -1,6 +1,6 @@
-from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 from pathlib import Path
-import pooch 
+
+from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 
 ###Metadata
 __version__ = 1
@@ -11,20 +11,16 @@
 ORIENTATION = "lps"
 ROOT_ID = 999  # Placeholder as no hierarchy is present
 RESOLUTION = 500  # um
-ATLAS_PACKAGER = 'Henry Crosswell'
+ATLAS_PACKAGER = "Henry Crosswell"
 
 
 def download_resources():
-
-
     """
     Download the necessary resources for the atlas.
 
     If possible, please use the Pooch library to retrieve any resources.
     """
 
-
-    
     pass
 
 
@@ -33,7 +29,7 @@ def retrieve_template_and_reference():
     Retrieve the desired template and reference as two numpy arrays.
 
     Returns:
-        tuple: A tuple containing two numpy arrays. The first array is the template volume, 
+        tuple: A tuple containing two numpy arrays. The first array is the template volume,
         and the second array is the reference volume.
     """
     template = None
@@ -45,14 +41,14 @@ def retrieve_hemisphere_map():
     """
     Retrieve a hemisphere map for the atlas.
 
-    If your atlas is asymmetrical, you may want to use a hemisphere map. 
+    If your atlas is asymmetrical, you may want to use a hemisphere map.
     This is an array in the same shape as your template,
     with 0's marking the left hemisphere, and 1's marking the right.
 
     If your atlas is symmetrical, ignore this function.
 
     Returns:
-        numpy.array or None: A numpy array representing the hemisphere map, 
+        numpy.array or None: A numpy array representing the hemisphere map,
         or None if the atlas is symmetrical.
     """
     return None

From 84e94f6ff4f9d02354bac5aef9f203ef5c29e3ee Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sun, 8 Sep 2024 13:06:54 +0100
Subject: [PATCH 05/22] formatted for black

---
 .../atlas_generation/atlas_scripts/catlas.py  | 56 +++++++++----------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index b2d7880a..0697491c 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -1,30 +1,28 @@
-from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 from pathlib import Path
-import pooch 
+
+from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 
 ###Metadata
 __version__ = 1
 ATLAS_NAME = "catlas"
-CITATION = "https://doi.org/10.1002/cne.24271"
+CITATION = "Stolzberg, Daniel et al 2017.https://doi.org/10.1002/cne.24271"
 SPECIES = "Felis catus"
 ATLAS_LINK = "https://github.com/CerebralSystemsLab/CATLAS"
 ORIENTATION = "lps"
 ROOT_ID = 999  # Placeholder as no hierarchy is present
 RESOLUTION = 500  # um
-ATLAS_PACKAGER = 'Henry Crosswell'
-
+ATLAS_PACKAGER = "Henry Crosswell"
 
-def download_resources():
+annotated_volume = None
+working_dir = None
 
 
+def download_resources():
     """
     Download the necessary resources for the atlas.
 
     If possible, please use the Pooch library to retrieve any resources.
     """
-
-
-    
     pass
 
 
@@ -33,7 +31,8 @@ def retrieve_template_and_reference():
     Retrieve the desired template and reference as two numpy arrays.
 
     Returns:
-        tuple: A tuple containing two numpy arrays. The first array is the template volume, 
+        tuple: A tuple containing two numpy arrays.
+        The first array is the template volume,
         and the second array is the reference volume.
     """
     template = None
@@ -45,14 +44,14 @@ def retrieve_hemisphere_map():
     """
     Retrieve a hemisphere map for the atlas.
 
-    If your atlas is asymmetrical, you may want to use a hemisphere map. 
+    If your atlas is asymmetrical, you may want to use a hemisphere map.
     This is an array in the same shape as your template,
     with 0's marking the left hemisphere, and 1's marking the right.
 
     If your atlas is symmetrical, ignore this function.
 
     Returns:
-        numpy.array or None: A numpy array representing the hemisphere map, 
+        numpy.array or None: A numpy array representing the hemisphere map,
         or None if the atlas is symmetrical.
     """
     return None
@@ -60,20 +59,21 @@ def retrieve_hemisphere_map():
 
 def retrieve_structure_information():
     """
-    This function should return a pandas DataFrame with information about your atlas.
+    This function should return a pandas DataFrame
+    with information about your atlas.
 
     The DataFrame should be in the following format:
 
-    ╭─────────────┬───────────────────────────────────┬─────────┬───────────────────────┬───────────────────╮
-    | id          | name                              | acronym | structure_id_path     | rgb_triplet       |
-    |             |                                   |         |                       |                   |
-    ├─────────────┼───────────────────────────────────┼─────────┼───────────────────────┼───────────────────┤
-    | 997         | root                              | root    | []                    | [255, 255, 255]   |
-    ├─────────────┼───────────────────────────────────┼─────────┼───────────────────────┼───────────────────┤
-    | 8           | Basic cell groups and regions     | grey    | [997]                 | [191, 218, 227]   |
-    ├─────────────┼───────────────────────────────────┼─────────┼───────────────────────┼───────────────────┤
-    | 567         | Cerebrum                          | CH      | [997, 8]              | [176, 240, 255]   |
-    ╰─────────────┴───────────────────────────────────┴─────────┴───────────────────────┴───────────────────╯
+    ╭─────┬──────────────────┬─────────┬───────────────────┬─────────────────╮
+    | id  | name             | acronym | structure_id_path | rgb_triplet     |
+    |     |                  |         |                   |                 |
+    ├─────┼──────────────────┼─────────┼───────────────────┼─────────────────┤
+    | 997 | root             | root    | []                | [255, 255, 255] |
+    ├─────┼──────────────────┼─────────┼───────────────────┼─────────────────┤
+    | 8   | grps and regions | grey    | [997]             | [191, 218, 227] |
+    ├─────┼──────────────────┼─────────┼───────────────────┼─────────────────┤
+    | 567 | Cerebrum         | CH      | [997, 8]          | [176, 240, 255] |
+    ╰─────┴──────────────────┴─────────┴───────────────────┴─────────────────╯
 
     Returns:
         pandas.DataFrame: A DataFrame containing the atlas information.
@@ -83,16 +83,16 @@ def retrieve_structure_information():
 
 def retrieve_or_construct_meshes():
     """
-    This function should return a dictionary of ids and corresponding paths to mesh files.
-    Some atlases are packaged with mesh files, in these cases we should use these files.
-    Then this function should download those meshes. In other cases we need to construct
-    the meshes ourselves. For this we have helper functions to achieve this.
+    This should return a dict of ids and corresponding paths to mesh files.
+    Use packaged mesh files if possible.
+    Download or construct mesh files  - use helper function for this
     """
     meshes_dict = {}
     return meshes_dict
 
 
-### If the code above this line has been filled correctly, nothing needs to be edited below (unless variables need to be passed between the functions).
+### If the code above this line has been filled correctly, nothing needs to be
+# edited below (unless variables need to be passed between the functions).
 bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
 bg_root_dir.mkdir(exist_ok=True)
 download_resources()

From 724531acd72753f8536d1efcc921d92e6c3b5aae Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sun, 8 Sep 2024 17:35:26 +0100
Subject: [PATCH 06/22] modified for precommit

---
 .../atlas_generation/atlas_scripts/catlas.py  | 103 ++++++++++++------
 1 file changed, 69 insertions(+), 34 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 5e478df9..187f5968 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -1,29 +1,61 @@
+""" Atlas for a domestic cat """
+
 from pathlib import Path
 
-from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
+import pooch
+
+from brainglobe_atlasapi.utils import check_internet_connection
 
 ###Metadata
 __version__ = 1
 ATLAS_NAME = "catlas"
 CITATION = "Stolzberg, Daniel et al 2017.https://doi.org/10.1002/cne.24271"
 SPECIES = "Felis catus"
-ATLAS_LINK = "https://github.com/CerebralSystemsLab/CATLAS"
+ATLAS_LINK = (
+    "https://github.com/CerebralSystemsLab/CATLAS/blob/main/SlicerFiles/"
+)
 ORIENTATION = "lps"
 ROOT_ID = 999  # Placeholder as no hierarchy is present
-RESOLUTION = 500  # um
+RESOLUTION = 500  # microns
 ATLAS_PACKAGER = "Henry Crosswell"
 
 annotated_volume = None
-working_dir = None
+working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
 
 
-def download_resources():
+def download_resources(working_dir):
     """
     Download the necessary resources for the atlas.
-
     If possible, please use the Pooch library to retrieve any resources.
     """
-    pass
+    # Setup download folder
+    download_dir_path = working_dir / "download_dir"
+    download_dir_path.mkdir(parents=True, exist_ok=True)
+    # Setup atlas folder within download_dir
+    atlas_dir_path = download_dir_path / "atlas_dir"
+    atlas_dir_path.mkdir(exist_ok=True)
+
+    check_internet_connection()
+    local_file_path_list = []
+    file_hash_list = [
+        ["meanBrain.nii", "md5:ffa42f5d703b192770d41cdf5493efc8"],
+        ["CorticalAtlas.nii", "md5:9b43e80b4052b7a8e8103163f4f5ff7d"],
+        ["CATLAS_COLORS.txt", "md5:d18c626858b492b139afc2094130b047"],
+        ["CorticalAtlas-Split.nii", "md5:c31fcaa92d658a20c3bb8059089bed14"],
+        ["CATLAS_COLORS-SPLIT.txt", "md5:bd7df732c51f23dae44daccbc58618bb"],
+    ]
+
+    for file, hash in file_hash_list:
+        file_path = atlas_dir_path / file
+        cached_file = pooch.retrieve(
+            url=ATLAS_LINK + file, known_hash=hash, path=file_path
+        )
+        local_file_path_list.append(cached_file)
+
+    return local_file_path_list
+
+
+print(download_resources(working_dir))
 
 
 def retrieve_template_and_reference():
@@ -93,32 +125,35 @@ def retrieve_or_construct_meshes():
     return meshes_dict
 
 
+# commenting out to unit test
+
 ### If the code above this line has been filled correctly, nothing needs to be
 # edited below (unless variables need to be passed between the functions).
-bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
-bg_root_dir.mkdir(exist_ok=True)
-download_resources()
-template_volume, reference_volume = retrieve_template_and_reference()
-hemispheres_stack = retrieve_hemisphere_map()
-structures = retrieve_structure_information()
-meshes_dict = retrieve_or_construct_meshes()
-
-output_filename = wrapup_atlas_from_data(
-    atlas_name=ATLAS_NAME,
-    atlas_minor_version=__version__,
-    citation=CITATION,
-    atlas_link=ATLAS_LINK,
-    species=SPECIES,
-    resolution=(RESOLUTION,) * 3,
-    orientation=ORIENTATION,
-    root_id=ROOT_ID,
-    reference_stack=template_volume,
-    annotation_stack=annotated_volume,
-    structures_list=structures,
-    meshes_dict=meshes_dict,
-    working_dir=working_dir,
-    hemispheres_stack=None,
-    cleanup_files=False,
-    compress=True,
-    scale_meshes=True,
-)
+# bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
+# bg_root_dir.mkdir(exist_ok=True)
+# working_dir = bg_root_dir
+# download_resources()
+# template_volume, reference_volume = retrieve_template_and_reference()
+# hemispheres_stack = retrieve_hemisphere_map()
+# structures = retrieve_structure_information()
+# meshes_dict = retrieve_or_construct_meshes()
+
+# output_filename = wrapup_atlas_from_data(
+#     atlas_name=ATLAS_NAME,
+#     atlas_minor_version=__version__,
+#     citation=CITATION,
+#     atlas_link=ATLAS_LINK,
+#     species=SPECIES,
+#     resolution=(RESOLUTION,) * 3,
+#     orientation=ORIENTATION,
+#     root_id=ROOT_ID,
+#     reference_stack=template_volume,
+#     annotation_stack=annotated_volume,
+#     structures_list=structures,
+#     meshes_dict=meshes_dict,
+#     working_dir=working_dir,
+#     hemispheres_stack=None,
+#     cleanup_files=False,
+#     compress=True,
+#     scale_meshes=True,
+# )

From b95d514b37426b70117e0e71891b5c69c5ad6eba Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sun, 8 Sep 2024 19:11:07 +0100
Subject: [PATCH 07/22] loaded raw data instead of url, loaded niftis into
 template and ref

---
 .../atlas_generation/atlas_scripts/catlas.py  | 48 ++++++++++---------
 1 file changed, 25 insertions(+), 23 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 187f5968..4f5e4495 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -3,6 +3,7 @@
 from pathlib import Path
 
 import pooch
+from brainglobe_utils.IO.image import load_nii
 
 from brainglobe_atlasapi.utils import check_internet_connection
 
@@ -11,15 +12,15 @@
 ATLAS_NAME = "catlas"
 CITATION = "Stolzberg, Daniel et al 2017.https://doi.org/10.1002/cne.24271"
 SPECIES = "Felis catus"
-ATLAS_LINK = (
-    "https://github.com/CerebralSystemsLab/CATLAS/blob/main/SlicerFiles/"
-)
+ATLAS_LINK = "https://github.com/CerebralSystemsLab/CATLAS"
+ATLAS_FILE_URL = "https://raw.githubusercontent.com/CerebralSystemsLab/CATLAS/main/SlicerFiles/"
 ORIENTATION = "lps"
 ROOT_ID = 999  # Placeholder as no hierarchy is present
 RESOLUTION = 500  # microns
 ATLAS_PACKAGER = "Henry Crosswell"
 
 annotated_volume = None
+# temp location
 working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
 
 
@@ -36,29 +37,28 @@ def download_resources(working_dir):
     atlas_dir_path.mkdir(exist_ok=True)
 
     check_internet_connection()
-    local_file_path_list = []
+    file_path_list = []
     file_hash_list = [
-        ["meanBrain.nii", "md5:ffa42f5d703b192770d41cdf5493efc8"],
-        ["CorticalAtlas.nii", "md5:9b43e80b4052b7a8e8103163f4f5ff7d"],
-        ["CATLAS_COLORS.txt", "md5:d18c626858b492b139afc2094130b047"],
-        ["CorticalAtlas-Split.nii", "md5:c31fcaa92d658a20c3bb8059089bed14"],
-        ["CATLAS_COLORS-SPLIT.txt", "md5:bd7df732c51f23dae44daccbc58618bb"],
+        ["meanBrain.nii", "md5:84e0d950474bd6c2a4bcebecd0e02ce7"],
+        ["CorticalAtlas.nii", "md5:942bbe2483c1d272434b4fd8f8df606f"],
+        ["CATLAS_COLORS.txt", "md5:5a48c961ebc1bbc2adb821be173b03e4"],
+        ["CorticalAtlas-Split.nii", "md5:7e883fefb60a289c70c4e5553c2c1f6a"],
+        ["CATLAS_COLORS-SPLIT.txt", "md5:ff80025b82b51c263ac2d1bfa3b8ae6b"],
     ]
 
     for file, hash in file_hash_list:
-        file_path = atlas_dir_path / file
         cached_file = pooch.retrieve(
-            url=ATLAS_LINK + file, known_hash=hash, path=file_path
+            url=ATLAS_FILE_URL + file, known_hash=hash, path=atlas_dir_path
         )
-        local_file_path_list.append(cached_file)
+        file_path_list.append(cached_file)
 
-    return local_file_path_list
+    return file_path_list
 
 
-print(download_resources(working_dir))
+file_path_list = download_resources(working_dir)
 
 
-def retrieve_template_and_reference():
+def retrieve_template_and_reference(file_path_list):
     """
     Retrieve the desired template and reference as two numpy arrays.
 
@@ -67,16 +67,18 @@ def retrieve_template_and_reference():
         The first array is the template volume,
         and the second array is the reference volume.
     """
-    template = None
-    reference = None
+    template = load_nii(file_path_list[0], as_array=True)
+    reference = load_nii(file_path_list[0], as_array=True)
     return template, reference
 
 
+template, reference = retrieve_template_and_reference(file_path_list)
+
+
 def retrieve_hemisphere_map():
     """
     Retrieve a hemisphere map for the atlas.
 
-    If your atlas is asymmetrical, you may want to use a hemisphere map.
     If your atlas is asymmetrical, you may want to use a hemisphere map.
     This is an array in the same shape as your template,
     with 0's marking the left hemisphere, and 1's marking the right.
@@ -84,7 +86,6 @@ def retrieve_hemisphere_map():
     If your atlas is symmetrical, ignore this function.
 
     Returns:
-        numpy.array or None: A numpy array representing the hemisphere map,
         numpy.array or None: A numpy array representing the hemisphere map,
         or None if the atlas is symmetrical.
     """
@@ -132,10 +133,11 @@ def retrieve_or_construct_meshes():
 # bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
 # bg_root_dir.mkdir(exist_ok=True)
 # working_dir = bg_root_dir
-# download_resources()
-# template_volume, reference_volume = retrieve_template_and_reference()
-# hemispheres_stack = retrieve_hemisphere_map()
-# structures = retrieve_structure_information()
+# local_file_path_list = download_resources(working_dir)
+# template_volume, reference_volume =
+# retrieve_template_and_reference(local_file_path_list)
+# hemispheres_stack = retrieve_hemisphere_map(local_file_path_list)
+# structures = retrieve_structure_information(local_file_path_list)
 # meshes_dict = retrieve_or_construct_meshes()
 
 # output_filename = wrapup_atlas_from_data(

From 43307a0697d468a9bbc1c7fc85bf7ab8258c8a22 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sun, 8 Sep 2024 19:19:21 +0100
Subject: [PATCH 08/22] changed reference and template slice index

---
 brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 4f5e4495..46d7ac9a 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -68,7 +68,7 @@ def retrieve_template_and_reference(file_path_list):
         and the second array is the reference volume.
     """
     template = load_nii(file_path_list[0], as_array=True)
-    reference = load_nii(file_path_list[0], as_array=True)
+    reference = load_nii(file_path_list[1], as_array=True)
     return template, reference
 
 

From 04173fe0a3109ef8b340da6d40e88723216fef98 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Mon, 9 Sep 2024 16:41:16 +0100
Subject: [PATCH 09/22] testing csv and text files for discrepancys before
 proceeding

---
 .../atlas_generation/atlas_scripts/catlas.py  | 24 ++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 46d7ac9a..5a490b13 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -2,6 +2,7 @@
 
 from pathlib import Path
 
+import pandas as pd
 import pooch
 from brainglobe_utils.IO.image import load_nii
 
@@ -23,6 +24,9 @@
 # temp location
 working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
 
+# A CSV I made from table 1 of the paper
+csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
+
 
 def download_resources(working_dir):
     """
@@ -92,7 +96,7 @@ def retrieve_hemisphere_map():
     return None
 
 
-def retrieve_structure_information():
+def retrieve_structure_information(file_path_list, csv_of_full_name):
     """
     This function should return a pandas DataFrame
     with information about your atlas.
@@ -113,9 +117,27 @@ def retrieve_structure_information():
     Returns:
         pandas.DataFrame: A DataFrame containing the atlas information.
     """
+
+    labels_df = pd.read_csv(file_path_list[2], sep=r"\s+", skiprows=2)
+    full_name_df = pd.read_csv(csv_of_full_name)
+
+    print(labels_df.shape)
+    print(full_name_df.shape)
+
+    col1 = labels_df.iloc[:, 1]
+    col2 = full_name_df.iloc[:, 0]
+
+    unique_in_df1 = set(col1) - set(col2)
+    unique_in_df2 = set(col2) - set(col1)
+
+    print(f"Values in df1 but not in df2: {unique_in_df1}")
+    print(f"Values in df2 but not in df1: {unique_in_df2}")
     return None
 
 
+retrieve_structure_information(file_path_list, csv_of_full_name)
+
+
 def retrieve_or_construct_meshes():
     """
     This should return a dict of ids and corresponding paths to mesh files.

From ebcb38dea3a11a34827c9acf7627925653183115 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Wed, 18 Sep 2024 18:12:11 +0100
Subject: [PATCH 10/22] changed for pre-commit formatting purposes

---
 .../atlas_generation/atlas_scripts/catlas.py  | 63 ++++++++++++++-----
 1 file changed, 48 insertions(+), 15 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 5a490b13..e844892a 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -24,7 +24,9 @@
 # temp location
 working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
 
-# A CSV I made from table 1 of the paper
+# A CSV I made from table 1 of the paper, cerebellum added
+# ALv and ALd included in catlas but not included on the table
+
 csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
 
 
@@ -96,6 +98,20 @@ def retrieve_hemisphere_map():
     return None
 
 
+def rgb_col_merger(labels_df):
+    """
+    Re-formats df columns, from individual r,g,b into the desired [r,g,b].
+    """
+
+    rgb_list = []
+    for _, row in labels_df.iterrows():
+        new_rgb_row = [row["r"], row["g"], row["b"]]
+        rgb_list.append(new_rgb_row)
+    labels_df = labels_df.drop(columns=["r", "g", "b"])
+    labels_df["rgb_triplet"] = rgb_list
+    return labels_df
+
+
 def retrieve_structure_information(file_path_list, csv_of_full_name):
     """
     This function should return a pandas DataFrame
@@ -118,21 +134,38 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         pandas.DataFrame: A DataFrame containing the atlas information.
     """
 
-    labels_df = pd.read_csv(file_path_list[2], sep=r"\s+", skiprows=2)
-    full_name_df = pd.read_csv(csv_of_full_name)
-
-    print(labels_df.shape)
-    print(full_name_df.shape)
-
-    col1 = labels_df.iloc[:, 1]
-    col2 = full_name_df.iloc[:, 0]
-
-    unique_in_df1 = set(col1) - set(col2)
-    unique_in_df2 = set(col2) - set(col1)
+    label_df_col = ["id", "acronym", "structure_id_path", "r", "g", "b"]
+    full_name_df_col = ["acronym", "name"]
+    combined_df_col = [
+        "id",
+        "name",
+        "acronym",
+        "structure_id_path",
+        "rgb_triplet",
+    ]
 
-    print(f"Values in df1 but not in df2: {unique_in_df1}")
-    print(f"Values in df2 but not in df1: {unique_in_df2}")
-    return None
+    labels_df = pd.read_csv(
+        file_path_list[2],
+        sep=r"\s+",
+        skiprows=2,
+        header=None,
+        names=label_df_col,
+        index_col=False,
+    )
+    full_name_df = pd.read_csv(
+        csv_of_full_name, names=full_name_df_col, index_col=False
+    )
+
+    # combines the last three [r],[g],[b] columns into one [r,g,b]
+    rgb_labels_df = rgb_col_merger(labels_df)
+
+    # merges both dataframes based aligning the fullnames with the acronyms,
+    # fills empty spaces with NaN and sorts df to desired structure.
+    structure_info_mix = pd.merge(
+        rgb_labels_df, full_name_df, on="acronym", how="left"
+    )
+    structure_info = structure_info_mix[combined_df_col]
+    return structure_info
 
 
 retrieve_structure_information(file_path_list, csv_of_full_name)

From f8f74af9d2ebf0608e4a13993e6e2e968584c8ef Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Wed, 18 Sep 2024 19:37:07 +0100
Subject: [PATCH 11/22] changed from structure, r,g,b to rgba, adding another
 function to account for structure id

---
 .../atlas_generation/atlas_scripts/catlas.py  | 36 +++++++------------
 1 file changed, 13 insertions(+), 23 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index e844892a..f1d7dadf 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -81,33 +81,23 @@ def retrieve_template_and_reference(file_path_list):
 template, reference = retrieve_template_and_reference(file_path_list)
 
 
-def retrieve_hemisphere_map():
-    """
-    Retrieve a hemisphere map for the atlas.
-
-    If your atlas is asymmetrical, you may want to use a hemisphere map.
-    This is an array in the same shape as your template,
-    with 0's marking the left hemisphere, and 1's marking the right.
-
-    If your atlas is symmetrical, ignore this function.
-
-    Returns:
-        numpy.array or None: A numpy array representing the hemisphere map,
-        or None if the atlas is symmetrical.
-    """
-    return None
+def add_heirarchy(labels_df):
+    # structure_id_path needs to be created, at the moment it is not.
+    heirarchy = []
+    labels_df["structure_id_path"] = heirarchy
+    pass
 
 
-def rgb_col_merger(labels_df):
+def add_rgb_col_and_heirarchy(labels_df):
     """
-    Re-formats df columns, from individual r,g,b into the desired [r,g,b].
+    Re-formats df columns, from individual r,g,b,a into the desired [r,g,b].
     """
 
     rgb_list = []
     for _, row in labels_df.iterrows():
         new_rgb_row = [row["r"], row["g"], row["b"]]
         rgb_list.append(new_rgb_row)
-    labels_df = labels_df.drop(columns=["r", "g", "b"])
+    labels_df = labels_df.drop(columns=["r", "g", "b", "alpha"])
     labels_df["rgb_triplet"] = rgb_list
     return labels_df
 
@@ -134,7 +124,7 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         pandas.DataFrame: A DataFrame containing the atlas information.
     """
 
-    label_df_col = ["id", "acronym", "structure_id_path", "r", "g", "b"]
+    label_df_col = ["id", "acronym", "r", "g", "b", "alpha"]
     full_name_df_col = ["acronym", "name"]
     combined_df_col = [
         "id",
@@ -156,15 +146,15 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         csv_of_full_name, names=full_name_df_col, index_col=False
     )
 
-    # combines the last three [r],[g],[b] columns into one [r,g,b]
-    rgb_labels_df = rgb_col_merger(labels_df)
+    complete_labels_df = add_rgb_col_and_heirarchy(labels_df)
 
-    # merges both dataframes based aligning the fullnames with the acronyms,
+    # merges both dataframes, aligning the fullnames with the acronyms,
     # fills empty spaces with NaN and sorts df to desired structure.
     structure_info_mix = pd.merge(
-        rgb_labels_df, full_name_df, on="acronym", how="left"
+        complete_labels_df, full_name_df, on="acronym", how="left"
     )
     structure_info = structure_info_mix[combined_df_col]
+    print(structure_info)
     return structure_info
 
 

From 72330bc250b8350f771fff4c59ff2b21d6d301f0 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Fri, 20 Sep 2024 11:44:05 +0100
Subject: [PATCH 12/22] edited function to create and add structure id, as well
 as adding a first line root to df

---
 .../atlas_generation/atlas_scripts/catlas.py  | 30 ++++++++++++++-----
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index f1d7dadf..db754fa6 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -81,11 +81,14 @@ def retrieve_template_and_reference(file_path_list):
 template, reference = retrieve_template_and_reference(file_path_list)
 
 
-def add_heirarchy(labels_df):
-    # structure_id_path needs to be created, at the moment it is not.
-    heirarchy = []
-    labels_df["structure_id_path"] = heirarchy
-    pass
+def add_heirarchy(labels_df_row):
+    """
+    Takes the index at a given row and adds the root_id -
+    produces structural heirarchy
+    """
+    structure_index = labels_df_row["id"]
+    structure_id = [ROOT_ID, structure_index]
+    return structure_id
 
 
 def add_rgb_col_and_heirarchy(labels_df):
@@ -94,11 +97,17 @@ def add_rgb_col_and_heirarchy(labels_df):
     """
 
     rgb_list = []
+    structure_list = []
     for _, row in labels_df.iterrows():
         new_rgb_row = [row["r"], row["g"], row["b"]]
         rgb_list.append(new_rgb_row)
+
+        structure_id = add_heirarchy(row)
+        structure_list.append(structure_id)
+
     labels_df = labels_df.drop(columns=["r", "g", "b", "alpha"])
     labels_df["rgb_triplet"] = rgb_list
+    labels_df["structure_id_path"] = structure_list
     return labels_df
 
 
@@ -133,6 +142,7 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         "structure_id_path",
         "rgb_triplet",
     ]
+    root_col = [col for col in combined_df_col if col != "name"]
 
     labels_df = pd.read_csv(
         file_path_list[2],
@@ -146,15 +156,21 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         csv_of_full_name, names=full_name_df_col, index_col=False
     )
 
-    complete_labels_df = add_rgb_col_and_heirarchy(labels_df)
+    labels_df = add_rgb_col_and_heirarchy(labels_df)
+    root_id_row = pd.DataFrame(
+        [[999, "root", [], [255, 255, 255]]], columns=root_col
+    )
+    complete_labels_df = pd.concat(
+        [root_id_row, labels_df.loc[:]]
+    ).reset_index(drop=True)
 
     # merges both dataframes, aligning the fullnames with the acronyms,
     # fills empty spaces with NaN and sorts df to desired structure.
     structure_info_mix = pd.merge(
         complete_labels_df, full_name_df, on="acronym", how="left"
     )
+
     structure_info = structure_info_mix[combined_df_col]
-    print(structure_info)
     return structure_info
 
 

From e4c8df0efd8b9a6a18978c7832b434ecdad13e0c Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sat, 28 Sep 2024 09:24:19 +0100
Subject: [PATCH 13/22] changed df into a dicitonary  list to work with the
 helper functions - create region mesh

---
 .../atlas_generation/atlas_scripts/catlas.py  | 33 +++++++++++++++----
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index db754fa6..f5507d4a 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -6,6 +6,7 @@
 import pooch
 from brainglobe_utils.IO.image import load_nii
 
+from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 from brainglobe_atlasapi.utils import check_internet_connection
 
 ###Metadata
@@ -67,6 +68,8 @@ def download_resources(working_dir):
 def retrieve_template_and_reference(file_path_list):
     """
     Retrieve the desired template and reference as two numpy arrays.
+    Template is MRI image of brain
+    Reference is an annotated 'segmentation' - each label has a unique ID
 
     Returns:
         tuple: A tuple containing two numpy arrays.
@@ -152,6 +155,7 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         names=label_df_col,
         index_col=False,
     )
+
     full_name_df = pd.read_csv(
         csv_of_full_name, names=full_name_df_col, index_col=False
     )
@@ -170,21 +174,38 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         complete_labels_df, full_name_df, on="acronym", how="left"
     )
 
-    structure_info = structure_info_mix[combined_df_col]
-    return structure_info
+    structures_df = structure_info_mix[combined_df_col]
+    structures_dict = structures_df.to_dict(orient="records")
+
+    hierarchy = get_structures_tree(structures_dict)
+    return hierarchy
 
 
-retrieve_structure_information(file_path_list, csv_of_full_name)
+hierarchy = retrieve_structure_information(file_path_list, csv_of_full_name)
 
 
-def retrieve_or_construct_meshes():
+def construct_meshes(hierarchy, reference_image):
     """
     This should return a dict of ids and corresponding paths to mesh files.
     Use packaged mesh files if possible.
     Download or construct mesh files  - use helper function for this
     """
-    meshes_dict = {}
-    return meshes_dict
+    pass
+
+
+#     )
+
+#     meshes_dir_path: pathlib Path object with folder where meshes are saved
+#     tree: treelib.Tree with hierarchical structures information
+#     node: tree's node corresponding to the region who's mesh is being created
+#     labels: list of unique label annotations in annotated volume,
+#     (list(np.unique(annotated_volume)))
+#     annotated_volume: 3d numpy array with annotaed volume
+#     ROOT_ID: int,
+#     id of root structure (mesh creation is a bit more refined for that)
+
+#     meshes_dict = {}
+#     return meshes_dict
 
 
 # commenting out to unit test

From b2dbf7d788d321204a462c0a8a67b9f70c1a1b6c Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sat, 28 Sep 2024 10:08:57 +0100
Subject: [PATCH 14/22] added the ability to make meshes, however it fails for
 about 10 ids, going to try and extract annotations from vtk files provided
 with data

---
 .../atlas_generation/atlas_scripts/catlas.py  | 76 ++++++++++++++++---
 1 file changed, 64 insertions(+), 12 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index f5507d4a..34295dd9 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -2,10 +2,18 @@
 
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
 import pooch
 from brainglobe_utils.IO.image import load_nii
-
+from rich.progress import track
+from skimage.filters.rank import modal
+from skimage.morphology import ball
+
+from brainglobe_atlasapi.atlas_generation.mesh_utils import (
+    Region,
+    create_region_mesh,
+)
 from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 from brainglobe_atlasapi.utils import check_internet_connection
 
@@ -65,11 +73,11 @@ def download_resources(working_dir):
 file_path_list = download_resources(working_dir)
 
 
-def retrieve_template_and_reference(file_path_list):
+def retrieve_template_and_annotations(file_path_list):
     """
-    Retrieve the desired template and reference as two numpy arrays.
+    Retrieve the desired template and annotations as two numpy arrays.
     Template is MRI image of brain
-    Reference is an annotated 'segmentation' - each label has a unique ID
+    Annotations is an annotated 'segmentation' - each label has a unique ID
 
     Returns:
         tuple: A tuple containing two numpy arrays.
@@ -77,11 +85,11 @@ def retrieve_template_and_reference(file_path_list):
         and the second array is the reference volume.
     """
     template = load_nii(file_path_list[0], as_array=True)
-    reference = load_nii(file_path_list[1], as_array=True)
-    return template, reference
+    annotations = load_nii(file_path_list[1], as_array=True)
+    return template, annotations
 
 
-template, reference = retrieve_template_and_reference(file_path_list)
+template, annotations = retrieve_template_and_annotations(file_path_list)
 
 
 def add_heirarchy(labels_df_row):
@@ -177,21 +185,65 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
     structures_df = structure_info_mix[combined_df_col]
     structures_dict = structures_df.to_dict(orient="records")
 
-    hierarchy = get_structures_tree(structures_dict)
-    return hierarchy
+    structures_tree = get_structures_tree(structures_dict)
+    return structures_tree
 
 
-hierarchy = retrieve_structure_information(file_path_list, csv_of_full_name)
+structures_tree = retrieve_structure_information(
+    file_path_list, csv_of_full_name
+)
 
 
-def construct_meshes(hierarchy, reference_image):
+def construct_meshes(structures_tree, annotations):
     """
     This should return a dict of ids and corresponding paths to mesh files.
     Use packaged mesh files if possible.
     Download or construct mesh files  - use helper function for this
     """
-    pass
+    # Generate binary mask for mesh creation
+    labels = np.unique(annotations).astype(np.int_)
+    for key, node in structures_tree.nodes.items():
+        # Check if the node's key is in the list of labels
+        is_label = key in labels
+        node.data = Region(is_label)
+
+    # Mesh creation parameters
+    closing_n_iters = 5
+    decimate_fraction = 0.6
+    smooth = True
+
+    meshes_dir_path = working_dir / "meshes"
+    meshes_dir_path.mkdir(exist_ok=True)
+
+    # pass a smoothed version of the annotations for meshing
+    smoothed_annotations = annotations.copy()
+    smoothed_annotations = modal(
+        smoothed_annotations.astype(np.uint8), ball(5)
+    )
+
+    # Iterate over each node in the tree and create meshes
+    for node in track(
+        structures_tree.nodes.values(),
+        total=structures_tree.size(),
+        description="Creating meshes",
+    ):
+
+        create_region_mesh(
+            [
+                meshes_dir_path,  # Directory where mesh files will be saved
+                node,
+                structures_tree,
+                labels,
+                smoothed_annotations,
+                ROOT_ID,
+                closing_n_iters,
+                decimate_fraction,
+                smooth,
+            ]
+        )
+
 
+construct_meshes(structures_tree, annotations)
 
 #     )
 

From 45a4d6cca2c32f85b32f41702c21f53567ac1cc3 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sat, 28 Sep 2024 16:50:35 +0100
Subject: [PATCH 15/22] created new function to take mesh from vtk files - uses
 a lot of temp locations, will endeaver to tidy

---
 .../atlas_generation/atlas_scripts/catlas.py  | 51 +++++++++++++++----
 1 file changed, 40 insertions(+), 11 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 34295dd9..dc12e16c 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -1,5 +1,6 @@
 """ Atlas for a domestic cat """
 
+import os
 from pathlib import Path
 
 import numpy as np
@@ -9,6 +10,7 @@
 from rich.progress import track
 from skimage.filters.rank import modal
 from skimage.morphology import ball
+from vedo import Mesh, write
 
 from brainglobe_atlasapi.atlas_generation.mesh_utils import (
     Region,
@@ -32,6 +34,10 @@
 annotated_volume = None
 # temp location
 working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
+mesh_folder_path = Path("/CATLAS-main/SlicerFiles/CorticalAtlasModel_A/")
+mesh_folder_path = working_dir + mesh_folder_path
+mesh_save_folder = working_dir / "meshes"
+
 
 # A CSV I made from table 1 of the paper, cerebellum added
 # ALv and ALd included in catlas but not included on the table
@@ -70,9 +76,6 @@ def download_resources(working_dir):
     return file_path_list
 
 
-file_path_list = download_resources(working_dir)
-
-
 def retrieve_template_and_annotations(file_path_list):
     """
     Retrieve the desired template and annotations as two numpy arrays.
@@ -89,9 +92,6 @@ def retrieve_template_and_annotations(file_path_list):
     return template, annotations
 
 
-template, annotations = retrieve_template_and_annotations(file_path_list)
-
-
 def add_heirarchy(labels_df_row):
     """
     Takes the index at a given row and adds the root_id -
@@ -186,12 +186,40 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
     structures_dict = structures_df.to_dict(orient="records")
 
     structures_tree = get_structures_tree(structures_dict)
-    return structures_tree
+    return structures_tree, structures_dict
 
 
-structures_tree = retrieve_structure_information(
-    file_path_list, csv_of_full_name
-)
+def extract_mesh_from_vtk(mesh_folder_path):
+    mesh_dict = {}
+    list_of_mesh_files = os.listdir(mesh_folder_path)
+    for vtk_file in list_of_mesh_files:
+        if vtk_file[-4:] != ".vtk":
+            continue
+        elif vtk_file == "A_32_Cerebelum.vtk":  # duplicate
+            continue
+        elif vtk_file == "A_36_pPE.vtk":  # duplicate & different acronym
+            continue
+        mesh = Mesh(mesh_folder_path + vtk_file)
+
+        # Re-creating the transformations from mesh_utils
+        mesh.triangulate()
+        mesh.decimate_pro(0.6)
+        mesh.smooth()
+
+        index = vtk_file[2:4]
+        if index[-1] == "_":
+            index = vtk_file[2]
+
+        file_name = index + ".obj"
+
+        if not Path(mesh_save_folder / file_name).exists():
+            write(mesh, str(mesh_save_folder / file_name))
+        else:
+            print(f"mesh already generated for file {index}")
+
+        mesh_dict[index] = file_name
+
+    return mesh_dict
 
 
 def construct_meshes(structures_tree, annotations):
@@ -243,7 +271,8 @@ def construct_meshes(structures_tree, annotations):
         )
 
 
-construct_meshes(structures_tree, annotations)
+# construct_meshes(structures_tree, annotations)
+
 
 #     )
 

From 9d2ccc01d7524fd25fa24d0fdb931235e86e8df2 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sat, 28 Sep 2024 16:52:32 +0100
Subject: [PATCH 16/22] remove unecessary mesh function

---
 .../atlas_generation/atlas_scripts/catlas.py  | 75 -------------------
 1 file changed, 75 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index dc12e16c..a2a98e21 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -3,19 +3,11 @@
 import os
 from pathlib import Path
 
-import numpy as np
 import pandas as pd
 import pooch
 from brainglobe_utils.IO.image import load_nii
-from rich.progress import track
-from skimage.filters.rank import modal
-from skimage.morphology import ball
 from vedo import Mesh, write
 
-from brainglobe_atlasapi.atlas_generation.mesh_utils import (
-    Region,
-    create_region_mesh,
-)
 from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 from brainglobe_atlasapi.utils import check_internet_connection
 
@@ -222,73 +214,6 @@ def extract_mesh_from_vtk(mesh_folder_path):
     return mesh_dict
 
 
-def construct_meshes(structures_tree, annotations):
-    """
-    This should return a dict of ids and corresponding paths to mesh files.
-    Use packaged mesh files if possible.
-    Download or construct mesh files  - use helper function for this
-    """
-    # Generate binary mask for mesh creation
-    labels = np.unique(annotations).astype(np.int_)
-    for key, node in structures_tree.nodes.items():
-        # Check if the node's key is in the list of labels
-        is_label = key in labels
-        node.data = Region(is_label)
-
-    # Mesh creation parameters
-    closing_n_iters = 5
-    decimate_fraction = 0.6
-    smooth = True
-
-    meshes_dir_path = working_dir / "meshes"
-    meshes_dir_path.mkdir(exist_ok=True)
-
-    # pass a smoothed version of the annotations for meshing
-    smoothed_annotations = annotations.copy()
-    smoothed_annotations = modal(
-        smoothed_annotations.astype(np.uint8), ball(5)
-    )
-
-    # Iterate over each node in the tree and create meshes
-    for node in track(
-        structures_tree.nodes.values(),
-        total=structures_tree.size(),
-        description="Creating meshes",
-    ):
-
-        create_region_mesh(
-            [
-                meshes_dir_path,  # Directory where mesh files will be saved
-                node,
-                structures_tree,
-                labels,
-                smoothed_annotations,
-                ROOT_ID,
-                closing_n_iters,
-                decimate_fraction,
-                smooth,
-            ]
-        )
-
-
-# construct_meshes(structures_tree, annotations)
-
-
-#     )
-
-#     meshes_dir_path: pathlib Path object with folder where meshes are saved
-#     tree: treelib.Tree with hierarchical structures information
-#     node: tree's node corresponding to the region who's mesh is being created
-#     labels: list of unique label annotations in annotated volume,
-#     (list(np.unique(annotated_volume)))
-#     annotated_volume: 3d numpy array with annotaed volume
-#     ROOT_ID: int,
-#     id of root structure (mesh creation is a bit more refined for that)
-
-#     meshes_dict = {}
-#     return meshes_dict
-
-
 # commenting out to unit test
 
 ### If the code above this line has been filled correctly, nothing needs to be

From 54912435a28afb5d0c777d062946e0b704f9f488 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Sun, 29 Sep 2024 18:27:04 +0100
Subject: [PATCH 17/22] edited structure id to work with wrapper function,
 removed unecessary text and functions

---
 .../atlas_generation/atlas_scripts/catlas.py  | 97 +++++++++----------
 1 file changed, 47 insertions(+), 50 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index a2a98e21..4ce63984 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -8,34 +8,22 @@
 from brainglobe_utils.IO.image import load_nii
 from vedo import Mesh, write
 
+from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 from brainglobe_atlasapi.utils import check_internet_connection
 
-###Metadata
+### Metadata
 __version__ = 1
-ATLAS_NAME = "catlas"
+ATLAS_NAME = "CSL_catlas"
 CITATION = "Stolzberg, Daniel et al 2017.https://doi.org/10.1002/cne.24271"
 SPECIES = "Felis catus"
 ATLAS_LINK = "https://github.com/CerebralSystemsLab/CATLAS"
 ATLAS_FILE_URL = "https://raw.githubusercontent.com/CerebralSystemsLab/CATLAS/main/SlicerFiles/"
 ORIENTATION = "lps"
-ROOT_ID = 999  # Placeholder as no hierarchy is present
+ROOT_ID = 997
 RESOLUTION = 500  # microns
 ATLAS_PACKAGER = "Henry Crosswell"
 
-annotated_volume = None
-# temp location
-working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
-mesh_folder_path = Path("/CATLAS-main/SlicerFiles/CorticalAtlasModel_A/")
-mesh_folder_path = working_dir + mesh_folder_path
-mesh_save_folder = working_dir / "meshes"
-
-
-# A CSV I made from table 1 of the paper, cerebellum added
-# ALv and ALd included in catlas but not included on the table
-
-csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
-
 
 def download_resources(working_dir):
     """
@@ -162,7 +150,7 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
 
     labels_df = add_rgb_col_and_heirarchy(labels_df)
     root_id_row = pd.DataFrame(
-        [[999, "root", [], [255, 255, 255]]], columns=root_col
+        [[997, "root", [997], [255, 255, 255]]], columns=root_col
     )
     complete_labels_df = pd.concat(
         [root_id_row, labels_df.loc[:]]
@@ -178,7 +166,7 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
     structures_dict = structures_df.to_dict(orient="records")
 
     structures_tree = get_structures_tree(structures_dict)
-    return structures_tree, structures_dict
+    return structures_tree, structures_dict, structures_df
 
 
 def extract_mesh_from_vtk(mesh_folder_path):
@@ -203,47 +191,56 @@ def extract_mesh_from_vtk(mesh_folder_path):
             index = vtk_file[2]
 
         file_name = index + ".obj"
+        file_path = str(mesh_save_folder / file_name)
+        mesh_dict[index] = file_path
 
-        if not Path(mesh_save_folder / file_name).exists():
-            write(mesh, str(mesh_save_folder / file_name))
+        if not Path(file_path).exists():
+            write(mesh, file_path)
         else:
-            print(f"mesh already generated for file {index}")
-
-        mesh_dict[index] = file_name
+            continue
 
     return mesh_dict
 
 
-# commenting out to unit test
+# A CSV I made from table 1 of the paper, cerebellum added
+# ALv and ALd included in catlas but not included on the table
+csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
+
+working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
+temp_dir = "F:/Users/Henry/Downloads/Setup/CATLAS-main/"
+mesh_folder_path = temp_dir + "CATLAS-main/SlicerFiles/CorticalAtlasModel_A/"
+mesh_save_folder = working_dir / "meshes"
 
-### If the code above this line has been filled correctly, nothing needs to be
+# If the code above this line has been filled correctly, nothing needs to be
 # edited below (unless variables need to be passed between the functions).
 # bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
 # bg_root_dir.mkdir(exist_ok=True)
 # working_dir = bg_root_dir
-# local_file_path_list = download_resources(working_dir)
-# template_volume, reference_volume =
-# retrieve_template_and_reference(local_file_path_list)
+local_file_path_list = download_resources(working_dir)
+template_volume, reference_volume = retrieve_template_and_annotations(
+    local_file_path_list
+)
 # hemispheres_stack = retrieve_hemisphere_map(local_file_path_list)
-# structures = retrieve_structure_information(local_file_path_list)
-# meshes_dict = retrieve_or_construct_meshes()
-
-# output_filename = wrapup_atlas_from_data(
-#     atlas_name=ATLAS_NAME,
-#     atlas_minor_version=__version__,
-#     citation=CITATION,
-#     atlas_link=ATLAS_LINK,
-#     species=SPECIES,
-#     resolution=(RESOLUTION,) * 3,
-#     orientation=ORIENTATION,
-#     root_id=ROOT_ID,
-#     reference_stack=template_volume,
-#     annotation_stack=annotated_volume,
-#     structures_list=structures,
-#     meshes_dict=meshes_dict,
-#     working_dir=working_dir,
-#     hemispheres_stack=None,
-#     cleanup_files=False,
-#     compress=True,
-#     scale_meshes=True,
-# )
+structures_tree, structures_dict, structures_df = (
+    retrieve_structure_information(local_file_path_list, csv_of_full_name)
+)
+meshes_dict = extract_mesh_from_vtk(mesh_folder_path)
+output_filename = wrapup_atlas_from_data(
+    atlas_name=ATLAS_NAME,
+    atlas_minor_version=__version__,
+    citation=CITATION,
+    atlas_link=ATLAS_LINK,
+    species=SPECIES,
+    resolution=(RESOLUTION,) * 3,
+    orientation=ORIENTATION,
+    root_id=ROOT_ID,
+    reference_stack=template_volume,
+    annotation_stack=reference_volume,
+    structures_list=structures_dict,
+    meshes_dict=meshes_dict,
+    working_dir=working_dir,
+    hemispheres_stack=None,
+    cleanup_files=False,
+    compress=True,
+    scale_meshes=True,
+)

From 1b5f08f424122f708ad675a10898c30722917012 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Wed, 2 Oct 2024 11:42:32 +0100
Subject: [PATCH 18/22] changed download resources to obtain annotations
 instead of relying on a local path

---
 .../atlas_generation/atlas_scripts/catlas.py  | 208 ++++++++++++++----
 1 file changed, 161 insertions(+), 47 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 4ce63984..86735cee 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -1,15 +1,18 @@
 """ Atlas for a domestic cat """
 
+import json
 import os
+import shutil
 from pathlib import Path
 
 import pandas as pd
 import pooch
+import requests
 from brainglobe_utils.IO.image import load_nii
+from bs4 import BeautifulSoup
 from vedo import Mesh, write
 
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
-from brainglobe_atlasapi.structure_tree_util import get_structures_tree
 from brainglobe_atlasapi.utils import check_internet_connection
 
 ### Metadata
@@ -27,35 +30,133 @@
 
 def download_resources(working_dir):
     """
-    Download the necessary resources for the atlas.
-    If possible, please use the Pooch library to retrieve any resources.
+    Downloads the nifti images, labels and annotations for the atlas.
+    Uses pooch hashes if they are present, if not obtains them
+
+    Returns:
+        List : list of all downloaded local filepaths
     """
     # Setup download folder
     download_dir_path = working_dir / "download_dir"
     download_dir_path.mkdir(parents=True, exist_ok=True)
+
     # Setup atlas folder within download_dir
     atlas_dir_path = download_dir_path / "atlas_dir"
-    atlas_dir_path.mkdir(exist_ok=True)
+    file_hash_jsonpath = atlas_dir_path / "hash_registry.json"
 
-    check_internet_connection()
+    file_path_and_hash_list = []
     file_path_list = []
-    file_hash_list = [
-        ["meanBrain.nii", "md5:84e0d950474bd6c2a4bcebecd0e02ce7"],
-        ["CorticalAtlas.nii", "md5:942bbe2483c1d272434b4fd8f8df606f"],
-        ["CATLAS_COLORS.txt", "md5:5a48c961ebc1bbc2adb821be173b03e4"],
-        ["CorticalAtlas-Split.nii", "md5:7e883fefb60a289c70c4e5553c2c1f6a"],
-        ["CATLAS_COLORS-SPLIT.txt", "md5:ff80025b82b51c263ac2d1bfa3b8ae6b"],
-    ]
+    check_internet_connection()
 
-    for file, hash in file_hash_list:
-        cached_file = pooch.retrieve(
-            url=ATLAS_FILE_URL + file, known_hash=hash, path=atlas_dir_path
+    if file_hash_jsonpath.exists():
+        print("Retrieving file paths and hash lists...")
+        hash_registry_json = atlas_dir_path / "hash_registry.json"
+        with open(hash_registry_json, "r") as json_file_to_edit:
+            file_path_and_hash_list = json.load(json_file_to_edit)
+
+    else:
+        # If dir is not already created - it downloads the data
+        print("Downloading file paths and hash lists...")
+        atlas_dir_path.mkdir(exist_ok=True)
+        file_names = [
+            "meanBrain.nii",
+            "CorticalAtlas.nii",
+            "CATLAS_COLORS.txt",
+        ]
+
+        file_hash_jsonpath = download_filename_and_hash(
+            file_names, atlas_dir_path
         )
-        file_path_list.append(cached_file)
 
+        hash_registry_json = download_multiple_filenames_and_hashs(
+            file_hash_jsonpath, atlas_dir_path
+        )
+
+        with open(hash_registry_json, "r") as json_file_to_edit:
+            file_path_and_hash_list = json.load(json_file_to_edit)
+
+    for file_path, _ in file_path_and_hash_list:
+        file_path_list.append(file_path)
     return file_path_list
 
 
+def download_filename_and_hash(file_names, atlas_dir_path):
+    """
+    Takes the given file names and uses pooch to download
+    the files converting into a json
+
+    Returns:
+        Json.file : a nested list containing [filepath,hash]
+    """
+    file_and_hash_list = []
+    file_hash_jsonpath = atlas_dir_path / "hash_registry.json"
+
+    for file in file_names:
+        cached_file = pooch.retrieve(
+            url=ATLAS_FILE_URL + file, known_hash=None, path=atlas_dir_path
+        )
+        file_hash = pooch.file_hash(cached_file, alg="md5")
+        file_and_hash_list.append([cached_file, file_hash])
+
+    with open(file_hash_jsonpath, "w") as file_and_hash_json:
+        json.dump(file_and_hash_list, file_and_hash_json)
+
+    return file_hash_jsonpath
+
+
+def download_multiple_filenames_and_hashs(file_hash_jsonpath, atlas_dir_path):
+    """
+    Opens the json file to write into, requests file names and uses these
+    to download the vtk annotation files, iterating through the folder
+
+    Returns:
+        Json.file : a updated nested list containing [filepath,hash]
+    """
+
+    vtk_folder_url = (
+        "https://github.com/CerebralSystemsLab/CATLAS/"
+        + "blob/main/SlicerFiles/CorticalAtlasModel_A/"
+    )
+
+    vtk_file_url = (
+        "https://github.com/CerebralSystemsLab/CATLAS/raw/"
+        + "refs/heads/main/SlicerFiles/CorticalAtlasModel_A/"
+    )
+
+    with open(file_hash_jsonpath, "r") as json_file_to_edit:
+        file_and_hash_list = json.load(json_file_to_edit)
+
+    response = requests.get(vtk_folder_url)
+    if response.status_code == 200:
+        soup = BeautifulSoup(response.content, "html.parser")
+        files = soup.find_all("a", class_="Link--primary")
+
+        seen_files = set()
+        for file in files:
+            file_name = file.get_text()
+
+            if file_name not in seen_files:
+                seen_files.add(file_name)
+
+                cached_file = pooch.retrieve(
+                    url=vtk_file_url + file_name,
+                    known_hash=None,
+                    path=atlas_dir_path,
+                )
+                file_hash = pooch.file_hash(cached_file, alg="md5")
+                file_and_hash_list.append([cached_file, file_hash])
+            else:
+                continue
+
+    else:
+        print("Incorrect URL given for VTK files")
+
+    with open(file_hash_jsonpath, "w") as file_and_hash_json:
+        json.dump(file_and_hash_list, file_and_hash_json)
+
+    return file_hash_jsonpath
+
+
 def retrieve_template_and_annotations(file_path_list):
     """
     Retrieve the desired template and annotations as two numpy arrays.
@@ -104,10 +205,10 @@ def add_rgb_col_and_heirarchy(labels_df):
 
 def retrieve_structure_information(file_path_list, csv_of_full_name):
     """
-    This function should return a pandas DataFrame
+    This function should return a list of dictionaries
     with information about your atlas.
 
-    The DataFrame should be in the following format:
+    The list of dictionaries should be in the following format:
 
     ╭─────┬──────────────────┬─────────┬───────────────────┬─────────────────╮
     | id  | name             | acronym | structure_id_path | rgb_triplet     |
@@ -121,7 +222,7 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
     ╰─────┴──────────────────┴─────────┴───────────────────┴─────────────────╯
 
     Returns:
-        pandas.DataFrame: A DataFrame containing the atlas information.
+        list of dicts: A list containing a dict of the atlas information.
     """
 
     label_df_col = ["id", "acronym", "r", "g", "b", "alpha"]
@@ -165,32 +266,48 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
     structures_df = structure_info_mix[combined_df_col]
     structures_dict = structures_df.to_dict(orient="records")
 
-    structures_tree = get_structures_tree(structures_dict)
-    return structures_tree, structures_dict, structures_df
+    return structures_dict
+
 
+def extract_mesh_from_vtk(working_dir):
+    """
+    Given the path to a folder containing annotation.vtk files
+    extracts the data as a vedo.mesh and saves as .obj
+    to be readable by other functions
 
-def extract_mesh_from_vtk(mesh_folder_path):
+    Returns:
+        dict: Key is obj id - value is obj file path
+    """
     mesh_dict = {}
-    list_of_mesh_files = os.listdir(mesh_folder_path)
+
+    atlas_dir_path = working_dir / "download_dir" / "atlas_dir"
+    mesh_save_folder = atlas_dir_path / "meshes"
+    mesh_save_folder.mkdir(parents=True, exist_ok=True)
+
+    list_of_mesh_files = os.listdir(atlas_dir_path)
+
     for vtk_file in list_of_mesh_files:
-        if vtk_file[-4:] != ".vtk":
-            continue
-        elif vtk_file == "A_32_Cerebelum.vtk":  # duplicate
+        if not vtk_file.endswith(".vtk"):
             continue
-        elif vtk_file == "A_36_pPE.vtk":  # duplicate & different acronym
+
+        # Checking for duplicates
+        elif vtk_file in ["A_32_Cerebelum.vtk", "A_36_pPE.vtk"]:
             continue
-        mesh = Mesh(mesh_folder_path + vtk_file)
+
+        mesh = Mesh(str(atlas_dir_path / vtk_file))
 
         # Re-creating the transformations from mesh_utils
         mesh.triangulate()
         mesh.decimate_pro(0.6)
         mesh.smooth()
 
-        index = vtk_file[2:4]
+        # Saves object files with a numerical index
+        index = str(vtk_file)
+        index = index.split("-")
+        index = index[1][2:4]
         if index[-1] == "_":
-            index = vtk_file[2]
-
-        file_name = index + ".obj"
+            index = index[0]
+        file_name = f"{index}.obj"
         file_path = str(mesh_save_folder / file_name)
         mesh_dict[index] = file_path
 
@@ -198,33 +315,30 @@ def extract_mesh_from_vtk(mesh_folder_path):
             write(mesh, file_path)
         else:
             continue
-
     return mesh_dict
 
 
-# A CSV I made from table 1 of the paper, cerebellum added
+# A CSV Table1 of paper, cerebellum added
 # ALv and ALd included in catlas but not included on the table
 csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
 
-working_dir = Path("F:/Users/Henry/Downloads/Setup/CATLAS-main/temp_pooch")
-temp_dir = "F:/Users/Henry/Downloads/Setup/CATLAS-main/"
-mesh_folder_path = temp_dir + "CATLAS-main/SlicerFiles/CorticalAtlasModel_A/"
-mesh_save_folder = working_dir / "meshes"
+bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
+
+# Deletes the folder since wrapup doesnt work if folder is created
+if bg_root_dir.exists():
+    shutil.rmtree(bg_root_dir)
 
-# If the code above this line has been filled correctly, nothing needs to be
-# edited below (unless variables need to be passed between the functions).
-# bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
-# bg_root_dir.mkdir(exist_ok=True)
-# working_dir = bg_root_dir
+bg_root_dir.mkdir(parents=True, exist_ok=True)
+working_dir = bg_root_dir
 local_file_path_list = download_resources(working_dir)
 template_volume, reference_volume = retrieve_template_and_annotations(
     local_file_path_list
 )
-# hemispheres_stack = retrieve_hemisphere_map(local_file_path_list)
-structures_tree, structures_dict, structures_df = (
-    retrieve_structure_information(local_file_path_list, csv_of_full_name)
+structures_dict = retrieve_structure_information(
+    local_file_path_list, csv_of_full_name
 )
-meshes_dict = extract_mesh_from_vtk(mesh_folder_path)
+meshes_dict = extract_mesh_from_vtk(working_dir)
+
 output_filename = wrapup_atlas_from_data(
     atlas_name=ATLAS_NAME,
     atlas_minor_version=__version__,

From 9e14c46955a9a45fe19996fd46dd8e578df0bc03 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Fri, 11 Oct 2024 16:37:41 +0100
Subject: [PATCH 19/22] hardcoded my csv of fullnames, mostly for testing and
 help on the code, can change back into csv if necessary

---
 .../atlas_generation/atlas_scripts/catlas.py  | 99 +++++++++++++++++--
 1 file changed, 93 insertions(+), 6 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 86735cee..2071fc23 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -245,9 +245,12 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
         index_col=False,
     )
 
-    full_name_df = pd.read_csv(
-        csv_of_full_name, names=full_name_df_col, index_col=False
-    )
+    # COMMENTED OUT CSV WORKFLOW AS HARDCODED INSTEAD
+    full_name_df = pd.DataFrame(csv_of_full_name, columns=full_name_df_col)
+
+    # full_name_df = pd.read_csv(
+    #     csv_of_full_name, names=full_name_df_col, index_col=False
+    # )
 
     labels_df = add_rgb_col_and_heirarchy(labels_df)
     root_id_row = pd.DataFrame(
@@ -318,9 +321,93 @@ def extract_mesh_from_vtk(working_dir):
     return mesh_dict
 
 
-# A CSV Table1 of paper, cerebellum added
-# ALv and ALd included in catlas but not included on the table
-csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
+# pre-commit wouldn't let me add this as a docstring
+
+# A copy of Table 1, manually copied from the paper.
+# Previously read from a csv I created, hard-coded for sharing purposes
+# the csv code will be commented out as we might
+# return to that method after testing.
+# Cerebellum added to full name csv, as not included in table 1
+# ALv and ALd are included in catlas text file but not included on the table,
+# replaced with NaN for full name - line 265
+
+
+# csv_of_full_name = "~/Desktop/catlas_table1_name.csv"
+csv_of_full_name = [
+    ["root", "root"],
+    ["A1", "Primary auditory cortex"],
+    ["A2", "Second auditory cortex"],
+    ["AAF", "Anterior auditory field"],
+    ["DP", "Posterior ectosylvian auditory cortex, dorsal division"],
+    ["DZ", "Dorsal zone of auditory cortex"],
+    ["EPp", "Posterior ectosylvian gyrus, posterior division"],
+    ["FAES", "FAES, Field of the anterior ectosylvian sulcus"],
+    ["IN", "Auditory insular cortex"],
+    ["iPE", "Posterior ectosylvian auditory cortex, intermediate division"],
+    ["PAF", "Posterior auditory field"],
+    ["TE", "Temporal cortex"],
+    ["VAF", "Ventral auditory field"],
+    ["vPAF", "Posterior auditory field, ventral division"],
+    ["vPE", "Posterior ectosylvian auditory cortex, ventral division"],
+    ["1", "Area 1, primary somatosensory cortex"],
+    ["2", "Area 2, primary somatosensory cortex"],
+    ["3a", "Area 3a primary somatosensory cortex"],
+    ["3b", "Area 3b primary somatosensory cortex"],
+    ["5al", "Area 5a, lateral division"],
+    ["5am", "Area 5a, medial division"],
+    ["5bl", "Area 5b, lateral division"],
+    ["5bm", "Area 5b, medial division"],
+    ["5m", "Area 5, medial division"],
+    ["S2", "Second somatosensory cortex"],
+    ["S2m", "Second somatosensory cortex, medial division"],
+    ["S3", "Third somatosensory cortex"],
+    ["S4", "Fourth somatosensory cortex"],
+    ["S5", "Fifth somatosensory cortex"],
+    ["17", "Area 17"],
+    ["18", "Area 18"],
+    ["19", "Area 19"],
+    ["20a", "Area 20a"],
+    ["20b", "Area 20b"],
+    ["21a", "Area 21a"],
+    ["21b", "Area 21b"],
+    ["7a", "Area 7"],
+    ["7m", "Area 7"],
+    ["7p", "Area 7"],
+    ["AEV", "Anterior ectosylvian visual area"],
+    ["ALLS", "Anterolateral lateral suprasylvian area"],
+    ["AMLS", "Anteromedial lateral suprasylvian area"],
+    ["CVA", "Cingulate visual area"],
+    ["DLS", "Dorsolateral suprasylvian visual area"],
+    ["PLLS", "Posterolateral lateral suprasylvian area"],
+    ["PMLS", "Posteromedial lateral suprasylvian area"],
+    ["PS", "Posterior suprasylvian visual area"],
+    ["SVA", "Splenial visual area"],
+    ["VLS", "Ventrolateral suprasylvian area"],
+    ["4Delta", "Area praecentralis macropyramidalis"],
+    ["4fu", "Area praecentralis in fundo"],
+    ["4Gamma", "Area praecentralis"],
+    ["4sfu", "Area praecentralis supra fundo"],
+    ["6aAlpha", "Area frontalis agranularis mediopyramidalis"],
+    ["6aBeta", "Area frontalis agranularis macropyramidalis"],
+    ["6aGamma", "Area 6, lateral division"],
+    ["6iffu", "Area 6, infra fundum"],
+    ["PFdl", "Prefrontal cortex, dorsolateral division"],
+    ["PFdm", "Prefrontal cortex, dorsomedial division"],
+    ["PFv", "Prefrontal cortex, ventral division"],
+    ["36", "Perirhinal cortex"],
+    ["AId", "Agranular insular area, dorsal division"],
+    ["AIv", "Agranular insular area, ventral division"],
+    ["DI", "Dysgranular insular area"],
+    ["GI", "Granular insular area"],
+    ["CGa", "Anterior cingulate area"],
+    ["CGp", "Posterior cingulate area"],
+    ["PL", "Prelimbic area"],
+    ["G", "Primary gustatory area"],
+    ["MZ", "Multisensory zone"],
+    ["Pp", "Prepyriform cortex"],
+    ["RS", "Retrosplenial area"],
+    ["Cbm", "Cerebellum"],
+]
 
 bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
 

From 0c0b4a15d42fba61acbf2eb34a1bd69be65a5e7e Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Fri, 22 Nov 2024 18:26:12 +0000
Subject: [PATCH 20/22] downscaled reference image and added a root for the
 mask, tidied up some docstrings and removed a redundant variable name

---
 .../atlas_generation/atlas_scripts/catlas.py  | 60 ++++++++++++++++---
 1 file changed, 52 insertions(+), 8 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 2071fc23..7bf64cc4 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -5,6 +5,7 @@
 import shutil
 from pathlib import Path
 
+import numpy as np
 import pandas as pd
 import pooch
 import requests
@@ -12,6 +13,9 @@
 from bs4 import BeautifulSoup
 from vedo import Mesh, write
 
+from brainglobe_atlasapi.atlas_generation.mesh_utils import (
+    extract_mesh_from_mask,
+)
 from brainglobe_atlasapi.atlas_generation.wrapup import wrapup_atlas_from_data
 from brainglobe_atlasapi.utils import check_internet_connection
 
@@ -133,6 +137,7 @@ def download_multiple_filenames_and_hashs(file_hash_jsonpath, atlas_dir_path):
 
         seen_files = set()
         for file in files:
+            print(file)
             file_name = file.get_text()
 
             if file_name not in seen_files:
@@ -160,17 +165,25 @@ def download_multiple_filenames_and_hashs(file_hash_jsonpath, atlas_dir_path):
 def retrieve_template_and_annotations(file_path_list):
     """
     Retrieve the desired template and annotations as two numpy arrays.
-    Template is MRI image of brain
-    Annotations is an annotated 'segmentation' - each label has a unique ID
+    Template_volume is an MRI image of brain
+    Reference_volume is an annotated segmentation - each label has a unique ID
 
     Returns:
         tuple: A tuple containing two numpy arrays.
-        The first array is the template volume,
-        and the second array is the reference volume.
+        The first array is a downscaled template volume,
+        and the second array is a reference volume.
     """
-    template = load_nii(file_path_list[0], as_array=True)
-    annotations = load_nii(file_path_list[1], as_array=True)
-    return template, annotations
+    template_volume = load_nii(file_path_list[0], as_array=True)
+    reference_volume = load_nii(file_path_list[1], as_array=True)
+
+    dmin = np.min(template_volume)
+    dmax = np.max(template_volume)
+    drange = dmax - dmin
+    dscale = (2**16 - 1) / drange
+    template_volume = (template_volume - dmin) * dscale
+    template_volume = template_volume.astype(np.uint16)
+
+    return template_volume, reference_volume
 
 
 def add_heirarchy(labels_df_row):
@@ -272,6 +285,35 @@ def retrieve_structure_information(file_path_list, csv_of_full_name):
     return structures_dict
 
 
+def create_mask_for_root(template_volume, reference_volume, mesh_save_folder):
+    """
+    A root_id mask of the mri image, with the annotations removed from it,
+    remaining is the root mask
+
+    returns:
+        a saved .obj file within mesh_save_folder
+    """
+
+    print(template_volume[100])
+    binarised_template_volume = np.where(
+        template_volume < 9000, 0, template_volume
+    )
+    binarised_template_volume = np.where(
+        binarised_template_volume != 0, ROOT_ID, binarised_template_volume
+    )
+    root_mask = np.where(reference_volume, 0, binarised_template_volume)
+
+    file_name = f"{ROOT_ID}.obj"
+    file_path = str(mesh_save_folder / file_name)
+
+    extract_mesh_from_mask(
+        root_mask,
+        obj_filepath=file_path,
+        smooth=True,
+        decimate_fraction=0.6,
+    )
+
+
 def extract_mesh_from_vtk(working_dir):
     """
     Given the path to a folder containing annotation.vtk files
@@ -288,7 +330,6 @@ def extract_mesh_from_vtk(working_dir):
     mesh_save_folder.mkdir(parents=True, exist_ok=True)
 
     list_of_mesh_files = os.listdir(atlas_dir_path)
-
     for vtk_file in list_of_mesh_files:
         if not vtk_file.endswith(".vtk"):
             continue
@@ -318,6 +359,9 @@ def extract_mesh_from_vtk(working_dir):
             write(mesh, file_path)
         else:
             continue
+
+    create_mask_for_root(template_volume, reference_volume, mesh_save_folder)
+    mesh_dict[ROOT_ID] = str(mesh_save_folder / f"{ROOT_ID}.obj")
     return mesh_dict
 
 

From ede19b55af787f626977141453bc72f553357b87 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Thu, 5 Dec 2024 20:49:46 +0000
Subject: [PATCH 21/22] updated download functions to properly use pooch
 hashes, when available. Also updated some variable names and docstrings to be
 more accurate

---
 .../atlas_generation/atlas_scripts/catlas.py  | 214 ++++++++++++------
 1 file changed, 141 insertions(+), 73 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index 7bf64cc4..b0eb672c 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -34,89 +34,138 @@
 
 def download_resources(working_dir):
     """
-    Downloads the nifti images, labels and annotations for the atlas.
-    Uses pooch hashes if they are present, if not obtains them
+    Checks to see if necessary files are already downloaded. If not,
+    downloads the nifti images, labels and annotations for the atlas.
+    Uses pooch hashes if they are present in hash_registry - creates
+    the json if it is not found
 
     Returns:
         List : list of all downloaded local filepaths
     """
     # Setup download folder
-    download_dir_path = working_dir / "download_dir"
-    download_dir_path.mkdir(parents=True, exist_ok=True)
+    temp_download_dir = working_dir / "download_dir"
 
-    # Setup atlas folder within download_dir
-    atlas_dir_path = download_dir_path / "atlas_dir"
-    file_hash_jsonpath = atlas_dir_path / "hash_registry.json"
+    # Hash_jsonpath needs to be external to the working dir, or it'll
+    # be overwritten - will need to change some variable names to make
+    # this work. For now it'll always be deleted and will therefore
+    # download the files without preset pooch hashes.
 
-    file_path_and_hash_list = []
+    # hash_jsonpath = "~/Desktop/hash_registry.json"
+    hash_jsonpath = working_dir / "hash_registry.json"
+    hash_json_exists = False
     file_path_list = []
     check_internet_connection()
 
-    if file_hash_jsonpath.exists():
-        print("Retrieving file paths and hash lists...")
-        hash_registry_json = atlas_dir_path / "hash_registry.json"
-        with open(hash_registry_json, "r") as json_file_to_edit:
-            file_path_and_hash_list = json.load(json_file_to_edit)
-
+    # Checks for json containing pooch registry
+    if hash_jsonpath.exists():
+        hash_json_exists = True
+        print("Retrieving file paths and pooch hashes...")
+        # Check to see if files are already downloaded
+        if not temp_download_dir.exists():
+            filepath_to_add, hash_to_add = download_files(
+                working_dir, hash_json_exists
+            )
+            filepath_jsonpath, hash_jsonpath = download_mesh_files(
+                filepath_to_add, hash_to_add, temp_download_dir
+            )
     else:
-        # If dir is not already created - it downloads the data
-        print("Downloading file paths and hash lists...")
-        atlas_dir_path.mkdir(exist_ok=True)
-        file_names = [
-            "meanBrain.nii",
-            "CorticalAtlas.nii",
-            "CATLAS_COLORS.txt",
-        ]
-
-        file_hash_jsonpath = download_filename_and_hash(
-            file_names, atlas_dir_path
+        # If json doesn't exist, it downloads the data and creates a json
+        print("Downloading file paths and pooch hashes...")
+        filepath_to_add, hash_to_add = download_files(
+            working_dir, hash_json_exists
         )
-
-        hash_registry_json = download_multiple_filenames_and_hashs(
-            file_hash_jsonpath, atlas_dir_path
+        filepath_jsonpath, hash_jsonpath = download_mesh_files(
+            filepath_to_add, hash_to_add, working_dir, hash_json_exists
         )
 
-        with open(hash_registry_json, "r") as json_file_to_edit:
-            file_path_and_hash_list = json.load(json_file_to_edit)
+    with open(filepath_jsonpath, "r") as json_file_to_edit:
+        filename_and_path_list = json.load(json_file_to_edit)
 
-    for file_path, _ in file_path_and_hash_list:
+    for _, file_path in filename_and_path_list:
         file_path_list.append(file_path)
+
     return file_path_list
 
 
-def download_filename_and_hash(file_names, atlas_dir_path):
+def download_files(working_dir, hash_json_exists):
     """
-    Takes the given file names and uses pooch to download
-    the files converting into a json
+    Takes pre-assigned file names and uses pooch to download
+    the files, writing filepath and pooch hash to json.
+    If Json containing the pooch hashes exists then it'll use them
 
     Returns:
-        Json.file : a nested list containing [filepath,hash]
+        filepath_to_add, hash_to_add :
+            path to a json file containing [filename, filepath]
+
+        hash_jsonpath :
+            path to a json file containing [filename, hash]
     """
-    file_and_hash_list = []
-    file_hash_jsonpath = atlas_dir_path / "hash_registry.json"
 
-    for file in file_names:
-        cached_file = pooch.retrieve(
-            url=ATLAS_FILE_URL + file, known_hash=None, path=atlas_dir_path
-        )
-        file_hash = pooch.file_hash(cached_file, alg="md5")
-        file_and_hash_list.append([cached_file, file_hash])
+    filename_hash_list = []
+    filename_filepath_list = []
+    temp_download_dir = working_dir / "download_dir"
+    temp_download_dir.mkdir(parents=True, exist_ok=True)
+
+    hash_jsonpath = working_dir / "hash_registry.json"
+    filepath_jsonpath = temp_download_dir / "path_registry.json"
+    file_names = [
+        "meanBrain.nii",
+        "CorticalAtlas.nii",
+        "CATLAS_COLORS.txt",
+    ]
+
+    # Loops through the files
+    for i, file in enumerate(file_names):
+
+        # If JSON exists, uses the pooch hashes to download the file
+        if hash_json_exists:
+            with open(hash_jsonpath, "r") as hash_json:
+                hash_json = json.load(hash_json)
+            hash = hash_json[i][1]
+            cached_file = pooch.retrieve(
+                url=ATLAS_FILE_URL + file,
+                known_hash=hash,
+                path=temp_download_dir,
+            )
+
+        else:
+            cached_file = pooch.retrieve(
+                url=ATLAS_FILE_URL + file,
+                known_hash=None,
+                path=temp_download_dir,
+            )
+            file_hash = pooch.file_hash(cached_file, alg="md5")
+            filename_hash_list.append([file, file_hash])
+            filename_filepath_list.append([file, cached_file])
 
-    with open(file_hash_jsonpath, "w") as file_and_hash_json:
-        json.dump(file_and_hash_list, file_and_hash_json)
+            with open(hash_jsonpath, "w") as hash_json:
+                json.dump(filename_hash_list, hash_json)
 
-    return file_hash_jsonpath
+        with open(filepath_jsonpath, "w") as filepath_json:
+            json.dump(filename_filepath_list, filepath_json)
 
+    return filepath_jsonpath, hash_jsonpath
 
-def download_multiple_filenames_and_hashs(file_hash_jsonpath, atlas_dir_path):
+
+def download_mesh_files(
+    filepath_to_add, hash_to_add, working_dir, hash_json_exists
+):
     """
-    Opens the json file to write into, requests file names and uses these
-    to download the vtk annotation files, iterating through the folder
+    Opens jsons from 'download files' function to add the
+    mesh filepath and hashes too. First checks to see if the hash_json
+    contains the necessary hashes for the download. If not then it
+    downloads the mesh files without and saves the hash to the hash_json.
 
     Returns:
-        Json.file : a updated nested list containing [filepath,hash]
+        filepath_to_add :
+            path to an updated json file containing [filename, filepath]
+        hash_to_add :
+            path to an updated json file containing [filename, hash]
     """
 
+    temp_download_dir = working_dir / "download_dir"
+    hash_jsonpath = working_dir / "hash_registry.json"
+
     vtk_folder_url = (
         "https://github.com/CerebralSystemsLab/CATLAS/"
         + "blob/main/SlicerFiles/CorticalAtlasModel_A/"
@@ -127,39 +176,56 @@ def download_multiple_filenames_and_hashs(file_hash_jsonpath, atlas_dir_path):
         + "refs/heads/main/SlicerFiles/CorticalAtlasModel_A/"
     )
 
-    with open(file_hash_jsonpath, "r") as json_file_to_edit:
-        file_and_hash_list = json.load(json_file_to_edit)
+    with open(filepath_to_add, "r") as filepath_to_edit:
+        filepath_list = json.load(filepath_to_edit)
+    with open(hash_to_add, "r") as hash_to_edit:
+        hash_list = json.load(hash_to_edit)
 
     response = requests.get(vtk_folder_url)
     if response.status_code == 200:
         soup = BeautifulSoup(response.content, "html.parser")
         files = soup.find_all("a", class_="Link--primary")
-
         seen_files = set()
+        index = 0
+
         for file in files:
-            print(file)
             file_name = file.get_text()
-
             if file_name not in seen_files:
                 seen_files.add(file_name)
-
-                cached_file = pooch.retrieve(
-                    url=vtk_file_url + file_name,
-                    known_hash=None,
-                    path=atlas_dir_path,
-                )
-                file_hash = pooch.file_hash(cached_file, alg="md5")
-                file_and_hash_list.append([cached_file, file_hash])
+                # If JSON exists, uses the pooch hashes for download
+                if hash_json_exists:
+                    with open(hash_jsonpath, "r") as hash_json:
+                        hash_json = json.load(hash_json)
+                    hash = hash_json[index][1]
+
+                    cached_file = pooch.retrieve(
+                        url=vtk_file_url + file_name,
+                        known_hash=hash,
+                        path=temp_download_dir,
+                    )
+                    index = +1
+
+                else:
+                    cached_file = pooch.retrieve(
+                        url=vtk_file_url + file_name,
+                        known_hash=None,
+                        path=temp_download_dir,
+                    )
+                    file_hash = pooch.file_hash(cached_file, alg="md5")
+                    hash_list.append([file_name, file_hash])
+                    filepath_list.append([file_name, cached_file])
             else:
                 continue
 
     else:
         print("Incorrect URL given for VTK files")
 
-    with open(file_hash_jsonpath, "w") as file_and_hash_json:
-        json.dump(file_and_hash_list, file_and_hash_json)
+    with open(hash_to_add, "w") as hash_json:
+        json.dump(hash_list, hash_json)
+    with open(filepath_to_add, "w") as filepath_json:
+        json.dump(filepath_list, filepath_json)
 
-    return file_hash_jsonpath
+    return filepath_to_add, hash_to_add
 
 
 def retrieve_template_and_annotations(file_path_list):
@@ -293,8 +359,6 @@ def create_mask_for_root(template_volume, reference_volume, mesh_save_folder):
     returns:
         a saved .obj file within mesh_save_folder
     """
-
-    print(template_volume[100])
     binarised_template_volume = np.where(
         template_volume < 9000, 0, template_volume
     )
@@ -325,11 +389,11 @@ def extract_mesh_from_vtk(working_dir):
     """
     mesh_dict = {}
 
-    atlas_dir_path = working_dir / "download_dir" / "atlas_dir"
-    mesh_save_folder = atlas_dir_path / "meshes"
+    temp_download_dir = working_dir / "download_dir"
+    mesh_save_folder = temp_download_dir / "meshes"
     mesh_save_folder.mkdir(parents=True, exist_ok=True)
 
-    list_of_mesh_files = os.listdir(atlas_dir_path)
+    list_of_mesh_files = os.listdir(temp_download_dir)
     for vtk_file in list_of_mesh_files:
         if not vtk_file.endswith(".vtk"):
             continue
@@ -338,7 +402,7 @@ def extract_mesh_from_vtk(working_dir):
         elif vtk_file in ["A_32_Cerebelum.vtk", "A_36_pPE.vtk"]:
             continue
 
-        mesh = Mesh(str(atlas_dir_path / vtk_file))
+        mesh = Mesh(str(temp_download_dir / vtk_file))
 
         # Re-creating the transformations from mesh_utils
         mesh.triangulate()
@@ -454,13 +518,14 @@ def extract_mesh_from_vtk(working_dir):
 ]
 
 bg_root_dir = Path.home() / "brainglobe_workingdir" / ATLAS_NAME
+working_dir = bg_root_dir
+temp_download_dir = bg_root_dir / "download_dir"
 
 # Deletes the folder since wrapup doesnt work if folder is created
 if bg_root_dir.exists():
     shutil.rmtree(bg_root_dir)
 
 bg_root_dir.mkdir(parents=True, exist_ok=True)
-working_dir = bg_root_dir
 local_file_path_list = download_resources(working_dir)
 template_volume, reference_volume = retrieve_template_and_annotations(
     local_file_path_list
@@ -488,4 +553,7 @@ def extract_mesh_from_vtk(working_dir):
     cleanup_files=False,
     compress=True,
     scale_meshes=True,
+    atlas_packager=ATLAS_PACKAGER,
 )
+
+shutil.rmtree(temp_download_dir)

From 7ec08856b911d131e92e28740656c5db920b3523 Mon Sep 17 00:00:00 2001
From: Henry Crosswell <henry.junkmail100@gmail.com>
Date: Thu, 5 Dec 2024 21:38:31 +0000
Subject: [PATCH 22/22] made sure the functionality behind the hash_json worked

---
 .../atlas_generation/atlas_scripts/catlas.py  | 26 +++++++++++++------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
index b0eb672c..e9592c0b 100644
--- a/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
+++ b/brainglobe_atlasapi/atlas_generation/atlas_scripts/catlas.py
@@ -50,8 +50,9 @@ def download_resources(working_dir):
     # this work. For now it'll always be deleted and will therefore
     # download the files without preset pooch hashes.
 
-    # hash_jsonpath = "~/Desktop/hash_registry.json"
-    hash_jsonpath = working_dir / "hash_registry.json"
+    hash_jsonpath = Path.home() / "hash_registry.json"
+    # hash_jsonpath = working_dir / "hash_registry.json"
+
     hash_json_exists = False
     file_path_list = []
     check_internet_connection()
@@ -66,7 +67,7 @@ def download_resources(working_dir):
                 working_dir, hash_json_exists
             )
             filepath_jsonpath, hash_jsonpath = download_mesh_files(
-                filepath_to_add, hash_to_add, temp_download_dir
+                filepath_to_add, hash_to_add, working_dir, hash_json_exists
             )
     else:
         # If json doesn't exist, it downloads the data and creates a json
@@ -106,7 +107,9 @@ def download_files(working_dir, hash_json_exists):
     temp_download_dir = working_dir / "download_dir"
     temp_download_dir.mkdir(parents=True, exist_ok=True)
 
-    hash_jsonpath = working_dir / "hash_registry.json"
+    # hash_jsonpath = working_dir / "hash_registry.json"
+    hash_jsonpath = Path.home() / "hash_registry.json"
+
     filepath_jsonpath = temp_download_dir / "path_registry.json"
     file_names = [
         "meanBrain.nii",
@@ -127,6 +130,7 @@ def download_files(working_dir, hash_json_exists):
                 known_hash=hash,
                 path=temp_download_dir,
             )
+            filename_filepath_list.append([file, cached_file])
 
         else:
             cached_file = pooch.retrieve(
@@ -135,8 +139,8 @@ def download_files(working_dir, hash_json_exists):
                 path=temp_download_dir,
             )
             file_hash = pooch.file_hash(cached_file, alg="md5")
+            file_hash = f"md5:{file_hash}"
             filename_hash_list.append([file, file_hash])
-            filename_filepath_list.append([file, cached_file])
 
             with open(hash_jsonpath, "w") as hash_json:
                 json.dump(filename_hash_list, hash_json)
@@ -163,8 +167,10 @@ def download_mesh_files(
             path to an updated json file containing [filename, hash]
     """
 
+    # hash_jsonpath = working_dir / "hash_registry.json"
+    hash_jsonpath = Path.home() / "hash_registry.json"
+
     temp_download_dir = working_dir / "download_dir"
-    hash_jsonpath = working_dir / "hash_registry.json"
 
     vtk_folder_url = (
         "https://github.com/CerebralSystemsLab/CATLAS/"
@@ -186,7 +192,7 @@ def download_mesh_files(
         soup = BeautifulSoup(response.content, "html.parser")
         files = soup.find_all("a", class_="Link--primary")
         seen_files = set()
-        index = 0
+        index = 3
 
         for file in files:
             file_name = file.get_text()
@@ -203,7 +209,8 @@ def download_mesh_files(
                         known_hash=hash,
                         path=temp_download_dir,
                     )
-                    index = +1
+                    filepath_list.append([file_name, cached_file])
+                    index += 1
 
                 else:
                     cached_file = pooch.retrieve(
@@ -212,6 +219,7 @@ def download_mesh_files(
                         path=temp_download_dir,
                     )
                     file_hash = pooch.file_hash(cached_file, alg="md5")
+                    file_hash = f"md5:{file_hash}"
                     hash_list.append([file_name, file_hash])
                     filepath_list.append([file_name, cached_file])
             else:
@@ -533,6 +541,8 @@ def extract_mesh_from_vtk(working_dir):
 structures_dict = retrieve_structure_information(
     local_file_path_list, csv_of_full_name
 )
+
+print("Converting VTK files into .obj mesh")
 meshes_dict = extract_mesh_from_vtk(working_dir)
 
 output_filename = wrapup_atlas_from_data(