Skip to content

Commit

Permalink
Merge pull request #1691 from girder/format-examples-datastore
Browse files Browse the repository at this point in the history 
Format Examples Pooch Datastore
  • Loading branch information
@annehaley
annehaley authored Oct 17, 2024
2 parents 45ef1e6 + b721b0e commit 91dcdb7
Showing 1 changed file with 40 additions and 24 deletions.
64 changes: 40 additions & 24 deletions docs/format_examples_datastore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
examples=[
dict(
filename='tcgaextract_ihergb_labeled.tiff',
url='https://data.kitware.com/api/v1/item/66e30827560e6127967912cf/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/faf5c8da95a5e624c70300afb98318e3421cc86ac27755f207075fa2f68aa23d099bec802007a86533579c6aadc97b4ce710d272eb871172d4b2c7e9ff6e9cad/download',
hash='8a03ae4370a35517e2ab2249177d903c1ea52869e30dc558750c626d9edbae6f',
),
],
Expand All @@ -25,7 +25,7 @@
examples=[
dict(
filename='CMU-1.svs',
url='https://data.kitware.com/api/v1/file/57b5d6558d777f10f2694486/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/3c31fd959302b56cf62020ed01312ab964bd776fbf7139a283bdde7c351d9211cec008eaf7f3b1b54e8e14c8293c6e6f77a389406e72318cb954f4a1a65f9ae1/download',
hash='00a3d54482cd707abf254fe69dccc8d06b8ff757a1663f1290c23418c480eb30',
),
],
Expand All @@ -38,7 +38,7 @@
examples=[
dict(
filename='DDX58_AXL_EGFR_well2_XY04.ome.tif',
url='https://data.kitware.com/api/v1/file/5fa1cbe750a41e3d192e18eb/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/d255bac7fdb3318ca015a688d396a2fc618a1383bc1751ec56100bf50b38f16d2d71a3837104fbe41bad524807123108e6fe0984f76bbf5e22656cea7541d44e/download',
hash='5dabb19081db4893d58b6680e54b566fc6246ccbde10d41413201afd96499482',
),
],
Expand All @@ -51,24 +51,24 @@
examples=[
dict(
filename='test3.ndpis',
url='https://data.kitware.com/api/v1/item/66e1af68560e612796791287/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/07fa23e39d1b79eb436aa282e340787e11ef32cd9547c246c0040f05bd0f0e6d283cf4107f31fdc0ef923d847d93941901e32885a163ba31da6c82f76d1b9744/download',
hash='35542dff44d7844f1fac04c1703bdb1476f90f484457907ead132203e6de066e',
),
dict(
filename='test3-TRITC 2 (560).ndpi',
url='https://data.kitware.com/api/v1/item/66e1af63560e61279679127e/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/6eae0f8bf57f42dee6a6f7f1da6d249fe06b11d7e3ac9c55721841c50961e4518b451bae9874d6aa767b1a53d4ac3bba3f72917b45cf21a8ddae8efb264a78a2/download',
skip=True,
hash='f02985d30aa38060e83c9477a4435cb81b2a0a234d3d664195dfc5a4e2657be8',
),
dict(
filename='test3-FITC 2 (485).ndpi',
url='https://data.kitware.com/api/v1/item/66e1af65560e612796791281/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/8f44a0a39a3a924ad831eb30ac5d906c601e9d16c79834edbe90affdf9c7ff54eea753075c117881ffb76d3ce542a18da3e4b262c87f4ec850e9fedd31b97385/download',
skip=True,
hash='455069a02a761ecfedb93492f103dec8c5b23b7c46fd47e9d8d22191ea803018',
),
dict(
filename='test3-DAPI 2 (387).ndpi',
url='https://data.kitware.com/api/v1/item/66e1af68560e612796791284/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/5ebdc2aed6ab277eb1644b9a7f489b70127b561ee8f9505677b1a3d3446d2d7a6ce10824d1e77ddd815a9a9214dc4423cb7de8539213265b5575e6ffc0d9e25f/download',
skip=True,
hash='9294d31f304f20ef1df1e08650d3f225b8ed44db8b2dc0d52f584449f85bdca2',
),
Expand All @@ -81,12 +81,12 @@
examples=[
dict(
filename='WD1.1_17-03_WT_MP.ome.zarr.zip',
url='https://data.kitware.com/api/v1/file/6544e995fdc1508d8bec1838/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/ebc026c279944e8d2829f3cfd12715cce73bd49f2d081688fd12bd758b26a2728a3f34f2c9c0c1b37f8bf822b5283908e84a41cb9b761000d5c8b43854572f32/download',
hash='f7dfb6683107d1cb21a360e31752b02641a16e0d32a87698669838418f108831',
),
dict(
filename='normmedia_8well_col2_livecellgfp.db',
url='https://data.kitware.com/api/v1/file/6544e9f5fdc1508d8bec1860/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/aa2c27d981ce01e75c642a68bee49752822eafdd68ca4eab0dffd0e04844499f7d3db18c21c59514b0b079e80a62cfc39c483d76c2ea962c45a9697de9e6fd08/download',
hash='7d6cfb6967f79eb5201eebbe93418e26445676d9ea95a3bf49ff4ffbf0993e39',
),
],
Expand All @@ -100,7 +100,7 @@
dict(
filename='sample_image.nd2',
# originally from 'https://downloads.openmicroscopy.org/images/ND2/aryeh/MeOh_high_fluo_003.nd2',
url='https://data.kitware.com/api/v1/item/66fde0be2480d44510d0ca24/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/4e76e490c915b10f646cb516f85a4d36d52aa7eff94715b90222644180e26fef6768493887c05adf182cf0351ba0bce659204041c4698a0f6b08423586788f4d/download',
hash='8e23bb594cd18314f9c18e70d736088ae46f8bc696ab7dc047784be416d7a706',
),
],
Expand All @@ -113,7 +113,7 @@
examples=[
dict(
filename='Plate1-Blue-A-02-Scene-1-P2-E1-01.czi',
url='https://data.kitware.com/api/v1/item/66e1af4d560e61279679127b/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/a707e0a65bbb743adb90ea17dd4e2d1ad1fbf7a105a29e8b67bd90b41525786136ed20ecc4876ff68abdceba8eb5f5bbedb9a364f5b260fa357d37987082a355/download',
hash='494936fa5a8c2e53b73672980986e2c44f3779e9b709ebfd406ffda723336376',
),
],
Expand All @@ -127,7 +127,7 @@
dict(
filename='US-MONO2-8-8x-execho.dcm',
# originally from 'https://downloads.openmicroscopy.org/images/DICOM/samples/US-MONO2-8-8x-execho.dcm',
url='https://data.kitware.com/api/v1/item/66fde1732480d44510d0ca27/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/5332044f887d82c7f3693c6ca180f07accf5f00c2b7b1a3a29ef9ae737d5f1975478b5e2d5846c391987b8051416068f57a7062e848323c700412236b35679db/download',
hash='7d3f54806d0315c6cfc8b7371649a242b5ef8f31e0d20221971dd8087f2ff1ea',
),
],
Expand All @@ -141,7 +141,7 @@
dict(
filename='20191025 Test FRET 585. 423, 426.lif',
# originally from 'https://downloads.openmicroscopy.org/images/Leica-LIF/imagesc-30856/20191025%20Test%20FRET%20585.%20423,%20426.lif',
url='https://data.kitware.com/api/v1/item/66fde1b82480d44510d0ca2a/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/d25de002d8a81dfcaf6b062b9f429ca85bb81423bc09d3aa33d9d51e9392cc4ace2b8521475e373ceecaf958effd0fade163e7173c467aab66c957da14482ed7/download',
hash='8d4ee62868b9616b832c2eb28e7d62ec050fb032e0bc11ea0a392f5c84390c71',
),
],
Expand All @@ -155,7 +155,7 @@
dict(
filename='Animated_PNG_example_bouncing_beach_ball.png',
# originally from 'https://upload.wikimedia.org/wikipedia/commons/1/14/Animated_PNG_example_bouncing_beach_ball.png',
url='https://data.kitware.com/api/v1/item/66fea4692480d44510d0ca6b/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/465ebdc2e81b2576dfc96b34e82db7f968e6d4f32f0fa80ef4bb0e44ed216230e6be1a2e4b11ae301a2905cc582dd24cbd2c360d9567ff7b1dac2c871f6d1e37/download',
hash='3b28e2462f1b31d0d15d795e6e58baf397899c3f864be7034bf47939b5bbbc3b',
),
],
Expand All @@ -172,7 +172,7 @@
),
dict(
filename='clouds.jpeg',
url='https://data.kitware.com/api/v1/file/5afd936c8d777f15ebe1b4d4/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/19796a50482aa834dca684129384c4bee87c43fb1636f0358211119b9274e05797db78cfa89192fc3395ad7cbb2d66e78e15ed8b8c9fed6361c0fc300d2fde93/download',
hash='fb195bb9dae13aa85d9f46f86ab9441a3b436dabbf57c68149453cba691d4391',
),
],
Expand All @@ -199,7 +199,7 @@
dict(
filename='SampleGIFImage_40kbmb.gif',
# originally from 'https://sample-videos.com/gif/3.gif',
url='https://data.kitware.com/api/v1/item/66fde6b52480d44510d0ca2d/download',
url='https://data.kitware.com/api/v1/file/hashsum/sha512/26d9a26b25a37f405b5c165b9ae338c873d27482e83881222df2af99e620b5f860a74ba92c0ca6caab2a45582ec5d82ba4c9a9c73870306c1eaa7d5fe69eafff/download',
hash='0ff064ba36e4f493f6a1b3d9d29c8eee1b719e39fc6768c5a6129534869c380b',
),
],
Expand All @@ -219,12 +219,28 @@
]


class ExamplesPooch(pooch.Pooch):
def get_url(self, fname):
self._assert_file_in_registry(fname)
for format_info in format_examples:
for example in format_info.get('examples', []):
if example.get('filename') == fname:
return example.get('url')


def fetch_all():
for format_data in format_examples:
for example in format_data.get('examples', []):
pooch.retrieve(
url=example.get('url'),
known_hash=example.get('hash'),
fname=example.get('filename'),
path=EXAMPLES_FOLDER,
)
registry = {}
for format_info in format_examples:
for example in format_info.get('examples', []):
filename = example.get('filename')
hash_value = example.get('hash')
if filename and hash_value:
registry[filename] = hash_value
datastore = ExamplesPooch(
path=EXAMPLES_FOLDER,
base_url='',
registry=registry,
retry_if_failed=10,
)
for key in registry:
datastore.fetch(key)

0 comments on commit 91dcdb7

Please sign in to comment.