From ca9d3d1f89c8a4c9cacb147ca4fcc2d44c6b26ab Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Thu, 9 Nov 2023 09:56:40 -0500 Subject: [PATCH 01/18] Add stacApiEndpoint and tileApiEndpoint to data mdx --- .../analysis/results/timeseries-data.ts | 7 +- .../common/blocks/scrollytelling/index.tsx | 1 + .../components/common/mapbox/index.tsx | 5 +- .../mapbox/layers/raster-timeseries.tsx | 19 +- .../mapbox/layers/vector-timeseries.tsx | 7 +- .../common/mapbox/layers/zarr-timeseries.tsx | 7 +- app/scripts/context/layer-data.tsx | 5 +- mock/datasets/data-from-ghg.data.mdx | 310 ++++++++++++++++++ 8 files changed, 347 insertions(+), 14 deletions(-) create mode 100644 mock/datasets/data-from-ghg.data.mdx diff --git a/app/scripts/components/analysis/results/timeseries-data.ts b/app/scripts/components/analysis/results/timeseries-data.ts index 9d8b16ba8..d3191fe2a 100644 --- a/app/scripts/components/analysis/results/timeseries-data.ts +++ b/app/scripts/components/analysis/results/timeseries-data.ts @@ -144,17 +144,18 @@ interface DatasetAssetsRequestParams { } async function getDatasetAssets( - { dateStart, dateEnd, stacCol, assets, aoi }: DatasetAssetsRequestParams, + { dateStart, dateEnd, stacApiEndpoint, stacCol, assets, aoi }: DatasetAssetsRequestParams, opts: AxiosRequestConfig, concurrencyManager: ConcurrencyManagerInstance ) { + const stacApiEndpointToUse = stacApiEndpoint?? process.env.API_STAC_ENDPOINT; const data = await concurrencyManager.queue(async () => { const collectionReqRes = await axios.get( - `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}` + `${stacApiEndpointToUse}/collections/${stacCol}` ); const searchReqRes = await axios.post( - `${process.env.API_STAC_ENDPOINT}/search`, + `${stacApiEndpointToUse}/search`, { 'filter-lang': 'cql2-json', limit: 10000, diff --git a/app/scripts/components/common/blocks/scrollytelling/index.tsx b/app/scripts/components/common/blocks/scrollytelling/index.tsx index e56873285..e30c18d39 100644 --- a/app/scripts/components/common/blocks/scrollytelling/index.tsx +++ b/app/scripts/components/common/blocks/scrollytelling/index.tsx @@ -455,6 +455,7 @@ function Scrollytelling(props) { key={runtimeData.id} id={runtimeData.id} mapInstance={mapRef.current} + stacCol={layer.stacApiEndpoint} stacCol={layer.stacCol} date={runtimeData.datetime} sourceParams={layer.sourceParams} diff --git a/app/scripts/components/common/mapbox/index.tsx b/app/scripts/components/common/mapbox/index.tsx index e63fc7eac..1258563aa 100644 --- a/app/scripts/components/common/mapbox/index.tsx +++ b/app/scripts/components/common/mapbox/index.tsx @@ -416,6 +416,8 @@ function MapboxMapComponent( BaseLayerComponent && ( { controller.abort(); }; - }, [mapInstance, id, stacCol, date, onStatusChange]); + }, [mapInstance, id, stacCol, stacApiEndpointToUse, date, onStatusChange]); const markerLayout = useCustomMarker(mapInstance); diff --git a/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx b/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx index 74c60799c..93651b97c 100644 --- a/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx +++ b/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx @@ -25,6 +25,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { const { id, stacCol, + stacApiEndpoint, date, mapInstance, sourceParams, @@ -39,6 +40,8 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { const [minZoom] = zoomExtent ?? [0, 20]; + const stacApiEndpointToUse = stacApiEndpoint?? process.env.API_STAC_ENDPOINT; + const generatorId = 'zarr-timeseries' + idSuffix; // @@ -51,7 +54,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { try { onStatusChange?.({ status: S_LOADING, id }); const data = await requestQuickCache({ - url: `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}`, + url: `${stacApiEndpointToUse}/collections/${stacCol}`, method: 'GET', controller }); @@ -72,7 +75,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { return () => { controller.abort(); }; - }, [mapInstance, id, stacCol, date, onStatusChange]); + }, [mapInstance, id, stacCol, stacApiEndpointToUse, date, onStatusChange]); // // Generate Mapbox GL layers and sources for raster timeseries diff --git a/app/scripts/context/layer-data.tsx b/app/scripts/context/layer-data.tsx index 20fa952aa..8c886a463 100644 --- a/app/scripts/context/layer-data.tsx +++ b/app/scripts/context/layer-data.tsx @@ -27,10 +27,11 @@ interface STACLayerData { const fetchLayerById = async ( layer: DatasetLayer | DatasetLayerCompareNormalized ): Promise => { - const { type, stacCol } = layer; + const { type, stacApiEndpoint, stacCol } = layer; + const stacApiEndpointToUse = stacApiEndpoint?? process.env.API_STAC_ENDPOINT; const { data } = await axios.get( - `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}` + `${stacApiEndpointToUse}/collections/${stacCol}` ); const commonTimeseriesParams = { diff --git a/mock/datasets/data-from-ghg.data.mdx b/mock/datasets/data-from-ghg.data.mdx new file mode 100644 index 000000000..2d266517f --- /dev/null +++ b/mock/datasets/data-from-ghg.data.mdx @@ -0,0 +1,310 @@ +--- +id: casagfed-carbonflux-monthgrid-v3 +name: CASA-GFED3 Land Carbon Flux +description: Global, monthly 0.5 degree resolution carbon fluxes from Net Primary Production (NPP), heterotrophic respiration (Rh), wildfire emissions (FIRE), and fuel wood burning emissions (FUEL) derived from the CASA-GFED model, version 3 +usage: + - url: 'https://us-ghg-center.github.io/ghgc-docs/cog_transformation/casagfed-carbonflux-monthgrid-v3.html' + label: Notebook showing data transformation to COG for ingest to the US GHG Center + title: 'Data Transformation Notebook' + - url: 'https://us-ghg-center.github.io/ghgc-docs/user_data_notebooks/casagfed-carbonflux-monthgrid-v3_User_Notebook.html' + label: Notebook to read, visualize, and explore data statistics + title: 'Sample Data Notebook' + - url: 'https://hub.ghg.center/hub/user-redirect/git-pull?repo=https%3A%2F%2Fgithub.com%2FUS-GHG-Center%2Fghgc-docs&urlpath=tree%2Fghgc-docs%2Fuser_data_notebooks%2Fcasagfed-carbonflux-monthgrid-v3_User_Notebook.ipynb&branch=main' + label: Run example notebook + title: Interactive Session in the US GHG Center JupyterHub (requires account) + - url: https://dljsq618eotzp.cloudfront.net/browseui/index.html#casagfed-carbonflux-monthgrid-v3/ + label: Browse and download the data + title: Data Browser +media: + src: ::file ./east_coast_mar_20.jpg + alt: wildfire + author: + name: Marcus Kauffman +taxonomy: + - name: Topics + values: + - Natural Emissions and Sinks + - name: Source + values: + - NASA + - name: Gas + values: + - CO₂ + - name: Product Type + values: + - Model Output +layers: + - id: casa-gfed-co2-flux + stacApiEndpoint: https://ghg.center/api/stac + tileApiEndpoint: https://ghg.center/api/raster + stacCol: casagfed-carbonflux-monthgrid-v3 + name: Net Primary Production (NPP) + type: raster + description: Model-estimated net primary production (NPP), which is the amount of carbon available from plants + initialDatetime: newest + projection: + id: 'equirectangular' + basemapId: 'light' + zoomExtent: + - 0 + - 20 + sourceParams: + assets: npp + colormap_name: purd + rescale: + - 0 + - 0.3 + compare: + datasetId: casagfed-carbonflux-monthgrid-v3 + layerId: casa-gfed-co2-flux + mapLabel: | + ::js ({ dateFns, datetime, compareDatetime }) => { + if (dateFns && datetime && compareDatetime) return `${dateFns.format(datetime, 'LLL yyyy')} VS ${dateFns.format(compareDatetime, 'LLL yyyy')}`; + } + legend: + unit: + label: kg Carbon/m²/mon + type: gradient + min: 0 + max: 0.3 + stops: + - '#F7F4F9' + - '#E9E3F0' + - '#D9C3DF' + - '#CDA0CD' + - '#D57ABA' + - '#E34A9F' + - '#DF2179' + - '#C10E51' + - '#92003F' + - '#67001F' + - id: casa-gfed-co2-flux-hr + stacApiEndpoint: https://ghg.center/api/stac + tileApiEndpoint: https://ghg.center/api/raster + stacCol: casagfed-carbonflux-monthgrid-v3 + name: Heterotrophic Respiration (Rh) + type: raster + description: Model-estimated heterotrophic respiration (Rh), which is the flux of carbon from the soil to the atmosphere + initialDatetime: newest + projection: + id: 'equirectangular' + basemapId: 'light' + zoomExtent: + - 0 + - 20 + sourceParams: + assets: rh + colormap_name: purd + rescale: + - 0 + - 0.3 + compare: + datasetId: casagfed-carbonflux-monthgrid-v3 + layerId: casa-gfed-co2-flux-hr + mapLabel: | + ::js ({ dateFns, datetime, compareDatetime }) => { + if (dateFns && datetime && compareDatetime) return `${dateFns.format(datetime, 'LLL yyyy')} VS ${dateFns.format(compareDatetime, 'LLL yyyy')}`; + } + legend: + unit: + label: kg Carbon/m²/mon + type: gradient + min: 0 + max: 0.3 + stops: + - '#F7F4F9' + - '#E9E3F0' + - '#D9C3DF' + - '#CDA0CD' + - '#D57ABA' + - '#E34A9F' + - '#DF2179' + - '#C10E51' + - '#92003F' + - '#67001F' + - id: casa-gfed-co2-flux-nee + stacApiEndpoint: https://ghg.center/api/stac + tileApiEndpoint: https://ghg.center/api/raster + stacCol: casagfed-carbonflux-monthgrid-v3 + name: Net Ecosystem Exchange (NEE) + type: raster + description: Model-estimated net ecosystem exchange (NEE), which is the net carbon flux to the atmosphere + initialDatetime: newest + projection: + id: 'equirectangular' + basemapId: 'light' + zoomExtent: + - 0 + - 20 + sourceParams: + assets: nee + colormap_name: coolwarm + rescale: + - -0.1 + - 0.1 + compare: + datasetId: casagfed-carbonflux-monthgrid-v3 + layerId: casa-gfed-co2-flux-nee + mapLabel: | + ::js ({ dateFns, datetime, compareDatetime }) => { + if (dateFns && datetime && compareDatetime) return `${dateFns.format(datetime, 'LLL yyyy')} VS ${dateFns.format(compareDatetime, 'LLL yyyy')}`; + } + legend: + unit: + label: kg Carbon/m²/mon + type: gradient + min: -0.1 + max: 0.1 + stops: + - '#3B4CC0' + - '#6788EE' + - '#9ABBFF' + - '#C9D7F0' + - '#EDD1C2' + - '#F7A889' + - '#E26952' + - '#B40426' + - id: casa-gfed-co2-flux-fe + stacApiEndpoint: https://ghg.center/api/stac + tileApiEndpoint: https://ghg.center/api/raster + stacCol: casagfed-carbonflux-monthgrid-v3 + name: Fire Emissions (FIRE) + type: raster + description: Model-estimated flux of carbon to the atmosphere from wildfires + initialDatetime: newest + projection: + id: 'equirectangular' + basemapId: 'light' + zoomExtent: + - 0 + - 20 + sourceParams: + assets: fire + colormap_name: purd + rescale: + - 0 + - 0.3 + compare: + datasetId: casagfed-carbonflux-monthgrid-v3 + layerId: casa-gfed-co2-flux-fe + mapLabel: | + ::js ({ dateFns, datetime, compareDatetime }) => { + if (dateFns && datetime && compareDatetime) return `${dateFns.format(datetime, 'LLL yyyy')} VS ${dateFns.format(compareDatetime, 'LLL yyyy')}`; + } + legend: + unit: + label: kg Carbon/m²/mon + type: gradient + min: 0 + max: 0.3 + stops: + - '#F7F4F9' + - '#E9E3F0' + - '#D9C3DF' + - '#CDA0CD' + - '#D57ABA' + - '#E34A9F' + - '#DF2179' + - '#C10E51' + - '#92003F' + - '#67001F' + - id: casa-gfed-co2-flux-fuel + stacApiEndpoint: https://ghg.center/api/stac + tileApiEndpoint: https://ghg.center/api/raster + stacCol: casagfed-carbonflux-monthgrid-v3 + name: Wood Fuel Emissions (FUEL) + type: raster + description: Model-estimated flux of carbon to the atmosphere from wood burned for fuel + initialDatetime: newest + projection: + id: 'equirectangular' + basemapId: 'light' + zoomExtent: + - 0 + - 20 + sourceParams: + assets: fuel + colormap_name: bupu + rescale: + - 0 + - 0.03 + compare: + datasetId: casagfed-carbonflux-monthgrid-v3 + layerId: casa-gfed-co2-flux-fuel + mapLabel: | + ::js ({ dateFns, datetime, compareDatetime }) => { + if (dateFns && datetime && compareDatetime) return `${dateFns.format(datetime, 'LLL yyyy')} VS ${dateFns.format(compareDatetime, 'LLL yyyy')}`; + } + legend: + unit: + label: kg Carbon/m²/mon + type: gradient + min: 0 + max: 0.03 + stops: + - '#F7FCFD' + - '#DCE9F2' + - '#B5CCE3' + - '#96ACD2' + - '#8C7DBA' + - '#894DA3' + - '#821580' + - '#4D004B' +--- + + + + This dataset presents a variety of carbon flux parameters derived from the Carnegie-Ames-Stanford-Approach – Global Fire Emissions Database version 3 (CASA-GFED3) model. The model’s input data includes air temperature, precipitation, incident solar radiation, a soil classification map, and a number of satellite derived products. All model calculations are driven by analyzed meteorological data from NASA’s Modern-Era Retrospective analysis for Research and Application, Version 2 (MERRA-2). The resulting product provides monthly, global data at 0.5 degree resolution from January 2003 through December 2017. It includes the following carbon flux variables expressed in units of kilograms of carbon per square meter per month (kg Carbon/m²/mon) from the following sources: net primary production (NPP), net ecosystem exchange (NEE), heterotrophic respiration (Rh), wildfire emissions (FIRE), and fuel wood burning emissions (FUEL). This product and earlier versions of MERRA-driven CASA-GFED carbon fluxes have been used in a number of atmospheric carbon dioxide (CO₂) transport studies, and through the support of NASA’s Carbon Monitoring System (CMS), it helps characterize, quantify, understand and predict the evolution of global carbon sources and sinks. + + - **Temporal Extent:** January 2003 - December 2017 + - **Temporal Resolution:** Monthly + - **Spatial Extent:** Global + - **Spatial Resolution:** 0.5° x 0.5° + - **Data Units:** Kilograms of carbon per square meter per month (kg Carbon/m²/mon) + - **Data Type:** Research + - **Data Latency:** Periodically updated when CASA-GFED model revised + + **Scientific Details:** Satellite derived products used as inputs for the CASA-GFED3 model include Moderate Resolution Imaging Spectroradiometer (MODIS) MOD12Q1 vegetation classification, MOD44B vegetation continuous fields, MOD09GA/MYD09GA based burned area, and Advanced Very High Resolution Radiometer (AVHRR) normalized difference vegetation index (NDVI). The fractional absorption of solar radiation by the vegetation canopy (FPAR), used for calculating NPP, was derived from Global Inventory Modeling and Mapping Studies (GIMMS) NDVI, produced from NOAA AVHRR data. This CASA-GFED3 dataset is a Version 3 data product that includes updates to the GIMMS NDVI input ([Pinzon & Tucker, 2014](https://doi.org/10.3390/rs6086929)) and uses the MODIS Collection 6 burned area mapping algorithm ([Giglio et al., 2018](https://doi.org/10.1016/j.rse.2018.08.005)). Also, additional flux variables that can be derived using this monthly product are listed below: + - NEP: monthly net ecosystem productivity, NEP = NPP - Rh + - NBP: monthly net biome productivity, net flux to the ecosystem, NBP = NPP - Rh - FIRE - FUEL + + + + + + ## Source Data Product Citation + Lesley Ott (2020), GEOS-Carb CASA-GFED Monthly Fire Fuel NPP Rh NEE Fluxes 0.5 degree x 0.5 degree V3, Greenbelt, MD, USA, Goddard Earth Sciences Data and Information Services Center (GES DISC), Accessed: [Data Access Date], [10.5067/03147VMJE8J9](https://doi.org/10.5067/03147VMJE8J9) + + ## Disclaimer + All data provided in the US GHG Center has been transformed from the original format (NetCDF) into Cloud Optimized GeoTIFF ([COG](https://www.cogeo.org/)). Careful quality checks are used to ensure data transformation has been performed correctly. + + The full title of this dataset, GEOS-Carb CASA-GFED Monthly Fire Fuel NPP Rh NEE Fluxes 0.5 degree x 0.5 degree V3, has been shortened for display on the US GHG Center website. The short name of the source dataset is GEOS_CASAGFED_M_FLUX, but it is referred to as casagfed-carbonflux-monthgrid-v3 within the Center system. The source dataset in NetCDF format is available from the [Goddard Earth Science Data and Information Services Center (GES DISC)](https://doi.org/10.5067/03147VMJE8J9). A user guide is available at [https://acdisc.gesdisc.eosdis.nasa.gov/data/CMS/GEOS_CASAGFED_M_FLUX.3/doc/README.CASA_GFED.pdf](https://acdisc.gesdisc.eosdis.nasa.gov/data/CMS/GEOS_CASAGFED_M_FLUX.3/doc/README.CASA_GFED.pdf) + + ## Key Publications + Ott, L., Collatz, J., & Kawa, R. (2020). *Description of GEOS-Carb CASA-GFED3 Land Carbon Flux Products*. GES DISC. [https://acdisc.gesdisc.eosdis.nasa.gov/data/CMS/GEOS_CASAGFED_M_FLUX.3/doc/README.CASA_GFED.pdf](https://acdisc.gesdisc.eosdis.nasa.gov/data/CMS/GEOS_CASAGFED_M_FLUX.3/doc/README.CASA_GFED.pdf) + + van der Werf, G. R., Randerson, J. T., Giglio, L., Collatz, G. J., Mu, M., Kasibhatla, P. S., Morton, D. C., DeFries, R. S., Jin, Y., & van Leeuwen, T. T. (2010). Global fire emissions and the contribution of deforestation, savanna, forest, agricultural, and peat fires (1997–2009). *Atmospheric Chemistry and Physics, 10*, 11707–11735. [https://doi.org/10.5194/acp-10-11707-2010](https://doi.org/10.5194/acp-10-11707-2010) + + ## Other Relevant Publications + Gelaro, R., McCarty, W., Suárez, M. J., Todling, R., Molod, A., Takacs, L., Randles, C. A., Darmenov, A., Bosilovich, M. G., Reichle, R., Wargan, K., Coy, L., Cullather, R., Draper, C., Akella, S., Buchard, V., Conaty, A., da Silva, A. M., Gu, W., … Zhao, B. (2017). The Modern-Era Retrospective Analysis for Research and Applications, Version 2 (MERRA-2). *Journal of Climate*, 30(14), 5419–5454. [https://doi.org/10.1175/jcli-d-16-0758.1](https://doi.org/10.1175/jcli-d-16-0758.1) + + Giglio, L., Boschetti, L., Roy, D. P., Humber, M. L., & Justice, C. O. (2018). The Collection 6 MODIS burned area mapping algorithm and product. *Remote Sensing of Environment*, 217, 72–85. [https://doi.org/10.1016/j.rse.2018.08.005](https://doi.org/10.1016/j.rse.2018.08.005) + + Ott, L. E., Pawson, S., Collatz, G. J., Gregg, W. W., Menemenlis, D., Brix, H., Rousseaux, C. S., Bowman, K. W., Liu, J., Eldering, A., Gunson, M. R., & Kawa, S. R. (2015). Assessing the magnitude of CO₂ flux uncertainty in atmospheric CO₂ records using products from NASA’s Carbon Monitoring Flux Pilot Project. *Journal of Geophysical Research: Atmospheres*, 120(2), 734–765. [https://doi.org/10.1002/2014jd022411](https://doi.org/10.1002/2014jd022411) + + Pinzon, J., & Tucker, C. (2014). A Non-Stationary 1981–2012 AVHRR NDVI3g Time Series. *Remote Sensing*, 6(8), 6929–6960. [https://doi.org/10.3390/rs6086929](https://doi.org/10.3390/rs6086929) + + van der Werf, G. R., Randerson, J. T., Giglio, L., van Leeuwen, T. T., Chen, Y., Rogers, B. M., Mu, M., van Marle, M. J. E., Morton, D. C., Collatz, G. J., Yokelson, R. J., & Kasibhatla, P. S. (2017). Global fire emissions estimates during 1997–2016. *Earth System Science Data*, 9, 697–720. [https://doi.org/10.5194/essd-9-697-2017](https://doi.org/10.5194/essd-9-697-2017) + + ## Acknowledgment + This dataset was produced as part of the [GEOS-Carb project](https://cce-datasharing.gsfc.nasa.gov/cmsprojects/list/h/0/) supported by NASA’s [Carbon Monitoring System (CMS) Program](https://carbon.nasa.gov/cms/). + + ## License + [Creative Commons Zero v1.0 Universal](https://creativecommons.org/publicdomain/zero/1.0/legalcode) (CC0 1.0) + + ## Data Stewardship + - [Data Workflow](https://us-ghg-center.github.io/ghgc-docs/data_workflow/casagfed-carbonflux-monthgrid-v3_Data_Flow.html) + - [Data Transformation Code](https://us-ghg-center.github.io/ghgc-docs/cog_transformation/casagfed-carbonflux-monthgrid-v3.html) + - [US GHG Center Data Intake Processing and Verification Report](https://us-ghg-center.github.io/ghgc-docs/processing_and_verification_reports/casagfed-carbonflux-monthgrid-v3_Processing%20and%20Verification%20Report.html) + + From 56dee23e12d31eaf5062279b75c8a6651175234f Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Thu, 9 Nov 2023 16:12:20 -0500 Subject: [PATCH 02/18] Add types, fix compare layer --- .../analysis/results/timeseries-data.ts | 1 + .../common/blocks/scrollytelling/index.tsx | 3 ++- app/scripts/components/common/mapbox/index.tsx | 8 +++++--- .../common/mapbox/layers/raster-timeseries.tsx | 15 ++++++++++----- .../components/common/mapbox/layers/utils.ts | 2 ++ .../common/mapbox/layers/vector-timeseries.tsx | 1 + .../common/mapbox/layers/zarr-timeseries.tsx | 11 ++++++----- parcel-resolver-veda/index.d.ts | 4 ++++ 8 files changed, 31 insertions(+), 14 deletions(-) diff --git a/app/scripts/components/analysis/results/timeseries-data.ts b/app/scripts/components/analysis/results/timeseries-data.ts index d3191fe2a..8626b594e 100644 --- a/app/scripts/components/analysis/results/timeseries-data.ts +++ b/app/scripts/components/analysis/results/timeseries-data.ts @@ -137,6 +137,7 @@ export function requestStacDatasetsTimeseries({ interface DatasetAssetsRequestParams { stacCol: string; + stacApiEndpoint?: string; assets: string; dateStart: Date; dateEnd: Date; diff --git a/app/scripts/components/common/blocks/scrollytelling/index.tsx b/app/scripts/components/common/blocks/scrollytelling/index.tsx index e30c18d39..795758656 100644 --- a/app/scripts/components/common/blocks/scrollytelling/index.tsx +++ b/app/scripts/components/common/blocks/scrollytelling/index.tsx @@ -455,7 +455,8 @@ function Scrollytelling(props) { key={runtimeData.id} id={runtimeData.id} mapInstance={mapRef.current} - stacCol={layer.stacApiEndpoint} + stacApiEndpoint={layer.stacApiEndpoint} + tileApiEndpoint={layer.tileApiEndpoint} stacCol={layer.stacCol} date={runtimeData.datetime} sourceParams={layer.sourceParams} diff --git a/app/scripts/components/common/mapbox/index.tsx b/app/scripts/components/common/mapbox/index.tsx index 1258563aa..67f03e3e3 100644 --- a/app/scripts/components/common/mapbox/index.tsx +++ b/app/scripts/components/common/mapbox/index.tsx @@ -228,7 +228,7 @@ function MapboxMapComponent( return [data, getLayerComponent(!!data.timeseries, data.type)]; }, [compareLayer, resolverBag]); - + // Get the compare to date. // The compare date is specified by the user. // If no date is specified anywhere we just use the same. @@ -471,14 +471,16 @@ function MapboxMapComponent( CompareLayerComponent && ( )} ; @@ -83,7 +85,7 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { } = props; const theme = useTheme(); - const { updateStyle } = useMapStyle(); + const { updateStyle } = useMapStyle(); const minZoom = zoomExtent?.[0] ?? 0; const generatorId = 'raster-timeseries' + idSuffix; @@ -148,7 +150,6 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { const [stacCollection, setStacCollection] = useState([]); useEffect(() => { if (!id || !stacCol || !date) return; - const controller = new AbortController(); const load = async () => { @@ -180,6 +181,7 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { payload, controller }); + console.log(responseData); /* eslint-disable no-console */ LOG && @@ -349,11 +351,14 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { useEffect( () => { const controller = new AbortController(); - + console.log('id'); + console.log(id); async function run() { + console.log('id'); + console.log(id); let layers: AnyLayer[] = []; let sources: Record = {}; - + console.log(mosaicUrl); if (mosaicUrl) { const tileParams = qs.stringify( { @@ -454,7 +459,7 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { }; layers = [...layers, pointsLayer]; } - + updateStyle({ generatorId, sources, diff --git a/app/scripts/components/common/mapbox/layers/utils.ts b/app/scripts/components/common/mapbox/layers/utils.ts index 97fecce74..4399859c9 100644 --- a/app/scripts/components/common/mapbox/layers/utils.ts +++ b/app/scripts/components/common/mapbox/layers/utils.ts @@ -134,6 +134,8 @@ export const getCompareLayerData = ( name: otherLayer.name, description: otherLayer.description, legend: otherLayer.legend, + stacApiEndpoint: otherLayer.stacApiEndpoint, + tileApiEndpoint: otherLayer.tileApiEndpoint, stacCol: otherLayer.stacCol, zoomExtent: zoomExtent ?? otherLayer.zoomExtent, sourceParams: defaultsDeep({}, sourceParams, otherLayer.sourceParams), diff --git a/app/scripts/components/common/mapbox/layers/vector-timeseries.tsx b/app/scripts/components/common/mapbox/layers/vector-timeseries.tsx index 92f236a2b..80e279707 100644 --- a/app/scripts/components/common/mapbox/layers/vector-timeseries.tsx +++ b/app/scripts/components/common/mapbox/layers/vector-timeseries.tsx @@ -22,6 +22,7 @@ import { userTzDate2utcString } from '$utils/date'; export interface MapLayerVectorTimeseriesProps { id: string; stacCol: string; + stacApiEndpoint?: string; date?: Date; mapInstance: MapboxMap; sourceParams?: Record; diff --git a/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx b/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx index 93651b97c..d1d964b22 100644 --- a/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx +++ b/app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx @@ -7,14 +7,14 @@ import { useMapStyle } from './styles'; import { ActionStatus, S_FAILED, S_LOADING, S_SUCCEEDED } from '$utils/status'; -const tilerUrl = process.env.API_XARRAY_ENDPOINT; - export interface MapLayerZarrTimeseriesProps { id: string; stacCol: string; date?: Date; mapInstance: MapboxMap; sourceParams?: Record; + stacApiEndpoint?: string; + tileApiEndpoint?: string; zoomExtent?: number[]; onStatusChange?: (result: { status: ActionStatus; id: string }) => void; isHidden?: boolean; @@ -25,7 +25,8 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { const { id, stacCol, - stacApiEndpoint, + stacApiEndpoint, + tileApiEndpoint, date, mapInstance, sourceParams, @@ -87,7 +88,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { useEffect( () => { - if (!tilerUrl) return; + if (!tileApiEndpoint) return; const tileParams = qs.stringify({ url: assetUrl, @@ -97,7 +98,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) { const zarrSource: RasterSource = { type: 'raster', - url: `${tilerUrl}?${tileParams}` + url: `${tileApiEndpoint}?${tileParams}` }; const zarrLayer: RasterLayer = { diff --git a/parcel-resolver-veda/index.d.ts b/parcel-resolver-veda/index.d.ts index 43ea624f0..6241abd1f 100644 --- a/parcel-resolver-veda/index.d.ts +++ b/parcel-resolver-veda/index.d.ts @@ -54,6 +54,8 @@ declare module 'veda' { export interface DatasetLayer extends DatasetLayerCommonProps { id: string; stacCol: string; + stacApiEndpoint?: string; + tileApiEndpoint?: string; name: string; description: string; initialDatetime?: 'newest' | 'oldest' | string; @@ -83,6 +85,8 @@ declare module 'veda' { id: string; name: string; description: string; + stacApiEndpoint?: string; + tileApiEndpoint?: string; stacCol: string; type: DatasetLayerType; legend?: LayerLegendCategorical | LayerLegendGradient; From 3f6c3d746585af53dc21d78160dbc74b46f43617 Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Fri, 10 Nov 2023 15:04:01 -0500 Subject: [PATCH 03/18] Add stacApiEndpoint to query key --- .../analysis/results/timeseries-data.ts | 12 ++++++++++-- app/scripts/context/layer-data.tsx | 16 ++++++++++------ 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/app/scripts/components/analysis/results/timeseries-data.ts b/app/scripts/components/analysis/results/timeseries-data.ts index 8626b594e..431203126 100644 --- a/app/scripts/components/analysis/results/timeseries-data.ts +++ b/app/scripts/components/analysis/results/timeseries-data.ts @@ -145,11 +145,18 @@ interface DatasetAssetsRequestParams { } async function getDatasetAssets( - { dateStart, dateEnd, stacApiEndpoint, stacCol, assets, aoi }: DatasetAssetsRequestParams, + { + dateStart, + dateEnd, + stacApiEndpoint, + stacCol, + assets, + aoi + }: DatasetAssetsRequestParams, opts: AxiosRequestConfig, concurrencyManager: ConcurrencyManagerInstance ) { - const stacApiEndpointToUse = stacApiEndpoint?? process.env.API_STAC_ENDPOINT; + const stacApiEndpointToUse = stacApiEndpoint ?? process.env.API_STAC_ENDPOINT; const data = await concurrencyManager.queue(async () => { const collectionReqRes = await axios.get( `${stacApiEndpointToUse}/collections/${stacCol}` @@ -240,6 +247,7 @@ async function requestTimeseries({ getDatasetAssets( { stacCol: layer.stacCol, + stacApiEndpoint: layer.stacApiEndpoint, assets: layer.sourceParams?.assets || 'cog_default', aoi, dateStart: start, diff --git a/app/scripts/context/layer-data.tsx b/app/scripts/context/layer-data.tsx index 8c886a463..0bebef1a0 100644 --- a/app/scripts/context/layer-data.tsx +++ b/app/scripts/context/layer-data.tsx @@ -28,7 +28,7 @@ const fetchLayerById = async ( layer: DatasetLayer | DatasetLayerCompareNormalized ): Promise => { const { type, stacApiEndpoint, stacCol } = layer; - const stacApiEndpointToUse = stacApiEndpoint?? process.env.API_STAC_ENDPOINT; + const stacApiEndpointToUse = stacApiEndpoint ?? process.env.API_STAC_ENDPOINT; const { data } = await axios.get( `${stacApiEndpointToUse}/collections/${stacCol}` @@ -69,7 +69,7 @@ const fetchLayerById = async ( const makeQueryObject = ( layer: DatasetLayer | DatasetLayerCompareNormalized ): UseQueryOptions => ({ - queryKey: ['layer', layer.stacCol], + queryKey: ['layer', layer.stacApiEndpoint, layer.stacCol], queryFn: () => fetchLayerById(layer), // This data will not be updated in the context of a browser session, so it is // safe to set the staleTime to Infinity. As specified by react-query's @@ -148,6 +148,7 @@ const useLayersInit = (layers: DatasetLayer[]): AsyncDatasetLayer[] => { /* eslint-disable-next-line @typescript-eslint/non-nullable-type-assertion-style */ const dataSTAC = queryClient.getQueryState([ 'layer', + baseData.stacApiEndpoint, baseData.stacCol ]) as QueryState; @@ -180,10 +181,13 @@ const useLayersInit = (layers: DatasetLayer[]): AsyncDatasetLayer[] => { baseLayer: mergeSTACData(layerProps), compareLayer: compareLayer && mergeSTACData(compareLayer), reFetch: () => - queryClient.refetchQueries(['layer', layer.stacCol], { - type: 'active', - exact: true - }) + queryClient.refetchQueries( + ['layer', layer.stacApiEndpoint, layer.stacCol], + { + type: 'active', + exact: true + } + ) }; }); }, [layers, queryClient, layerQueries]); From b1c735fe8ff71328890598f2fbda04054895352e Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Fri, 10 Nov 2023 16:35:45 -0500 Subject: [PATCH 04/18] Handle multi stac endpoints in analysis --- .../define/use-stac-collection-search.ts | 53 +++++++++++++------ .../analysis/results/timeseries-data.ts | 5 +- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/app/scripts/components/analysis/define/use-stac-collection-search.ts b/app/scripts/components/analysis/define/use-stac-collection-search.ts index 86e324a7f..b77188122 100644 --- a/app/scripts/components/analysis/define/use-stac-collection-search.ts +++ b/app/scripts/components/analysis/define/use-stac-collection-search.ts @@ -4,9 +4,7 @@ import axios from 'axios'; import { useQuery } from '@tanstack/react-query'; import booleanIntersects from '@turf/boolean-intersects'; import bboxPolygon from '@turf/bbox-polygon'; -import { - areIntervalsOverlapping -} from 'date-fns'; +import { areIntervalsOverlapping } from 'date-fns'; import { DatasetLayer } from 'veda'; import { MAX_QUERY_NUM } from '../constants'; @@ -25,8 +23,6 @@ interface UseStacSearchProps { export type DatasetWithTimeseriesData = TimeseriesDataResult & DatasetLayer & { numberOfItems: number }; -const collectionUrl = `${process.env.API_STAC_ENDPOINT}/collections`; - export function useStacCollectionSearch({ start, end, @@ -37,10 +33,31 @@ export function useStacCollectionSearch({ const result = useQuery({ queryKey: ['stacCollection'], queryFn: async ({ signal }) => { - const collectionResponse = await axios.get(collectionUrl, { - signal - }); - return collectionResponse.data.collections; + const collectionUrlsFromDataSets = allAvailableDatasetsLayers + .filter((dataset) => dataset.stacApiEndpoint) + .map((dataset) => `${dataset.stacApiEndpoint}/collections`) + .filter((value, index, array) => array.indexOf(value) === index); + + const collectionUrls = [ + ...collectionUrlsFromDataSets, + `${process.env.API_STAC_ENDPOINT}/collections` + ]; + + const colloectionRequests = collectionUrls.map((url: string) => + axios.get(url, { signal }).then((response) => { + return response.data.collections.map((col) => ({ + ...col, + stacApiEndpoint: url.replace(/\/collections$/, '') + })); + }) + ); + return axios.all(colloectionRequests).then( + axios.spread((...responses) => { + // Merge all responses into one array + const mergedData = [].concat(...responses); + return mergedData; + }) + ); }, enabled: readyToLoadDatasets }); @@ -86,13 +103,15 @@ export function useStacCollectionSearch({ function getInTemporalAndSpatialExtent(collectionData, aoi, timeRange) { const matchingCollectionIds = collectionData.reduce((acc, col) => { - const id = col.id; + const { id, stacEndpoint } = col; // Is is a dataset defined in the app? // If not, skip other calculations. - const isAppDataset = allAvailableDatasetsLayers.some( - (l) => l.stacCol === id - ); + const isAppDataset = allAvailableDatasetsLayers.some((l) => { + const stacApiEndpointUsed = + l.stacApiEndpoint ?? process.env.API_STAC_ENDPOINT; + return l.stacCol === id && stacApiEndpointUsed === stacEndpoint; + }); if ( !isAppDataset || @@ -106,7 +125,7 @@ function getInTemporalAndSpatialExtent(collectionData, aoi, timeRange) { const start = utcString2userTzDate(col.extent.temporal.interval[0][0]); const end = utcString2userTzDate(col.extent.temporal.interval[0][1]); - const isInAOI = aoi.features.some((feature) => + const isInAOI = aoi.features?.some((feature) => booleanIntersects(feature, bboxPolygon(bbox)) ); @@ -130,7 +149,11 @@ function getInTemporalAndSpatialExtent(collectionData, aoi, timeRange) { ); const filteredDatasetsWithCollections = filteredDatasets.map((l) => { - const collection = collectionData.find((c) => c.id === l.stacCol); + const stacApiEndpointUsed = + l.stacApiEndpoint ?? process.env.API_STAC_ENDPOINT; + const collection = collectionData.find( + (c) => c.id === l.stacCol && stacApiEndpointUsed === c.stacApiEndpoint + ); return { ...l, isPeriodic: collection['dashboard:is_periodic'], diff --git a/app/scripts/components/analysis/results/timeseries-data.ts b/app/scripts/components/analysis/results/timeseries-data.ts index 431203126..502f4b4a6 100644 --- a/app/scripts/components/analysis/results/timeseries-data.ts +++ b/app/scripts/components/analysis/results/timeseries-data.ts @@ -277,6 +277,9 @@ async function requestTimeseries({ } }); + const tileEndpointToUse = + layer.tileApiEndpoint ?? process.env.API_RASTER_ENDPOINT; + const layerStatistics = await Promise.all( assets.map(async ({ date, url }) => { const statistics = await queryClient.fetchQuery( @@ -284,7 +287,7 @@ async function requestTimeseries({ async ({ signal }) => { return concurrencyManager.queue(async () => { const { data } = await axios.post( - `${process.env.API_RASTER_ENDPOINT}/cog/statistics?url=${url}`, + `${tileEndpointToUse}/cog/statistics?url=${url}`, // Making a request with a FC causes a 500 (as of 2023/01/20) combineFeatureCollection(aoi), { signal } From cb762ac86c168cf2de0c25b1e6a760317378d1d6 Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Fri, 10 Nov 2023 16:36:21 -0500 Subject: [PATCH 05/18] Delete oco2 dataset for testing --- mock/datasets/oco2-geos-l3-daily.data.mdx | 46 ----------------------- 1 file changed, 46 deletions(-) delete mode 100644 mock/datasets/oco2-geos-l3-daily.data.mdx diff --git a/mock/datasets/oco2-geos-l3-daily.data.mdx b/mock/datasets/oco2-geos-l3-daily.data.mdx deleted file mode 100644 index dc5dd4227..000000000 --- a/mock/datasets/oco2-geos-l3-daily.data.mdx +++ /dev/null @@ -1,46 +0,0 @@ ---- -id: oco2-geos-l3-daily -name: 'Gridded Daily OCO-2 Carbon Dioxide assimilated dataset' -featured: true -description: "NASA’s Orbiting Carbon Observatory, 2 (OCO-2) provides the most complete dataset tracking the concentration of atmospheric carbon dioxide (CO₂), the main driver of climate change. Since its launch (July 2014), OCO-2 measures sunlight reflected from Earth’s surface to infer the dry-air column-averaged CO2 mixing ratio and provides around 100,000 cloud-free observations." -media: - src: ::file ./no2--dataset-cover.jpg - alt: Power plant shooting steam at the sky. - author: - name: Mick Truyts - url: https://unsplash.com/photos/x6WQeNYJC1w -taxonomy: - - name: Topics - values: - - Air Quality -layers: - - id: oco2-geos-l3-daily - stacCol: oco2-geos-l3-daily - name: Gridded Daily OCO-2 Carbon Dioxide assimilated dataset - type: zarr - description: - "The OCO-2 mission provides the highest quality space-based XCO2 retrievals to date. However, the instrument data are characterized by large gaps in coverage due to OCO-2’s narrow 10-km ground track and an inability to see through clouds and thick aerosols. This global gridded dataset is produced using a data assimilation technique commonly referred to as state estimation within the geophysical literature. Data assimilation synthesizes simulations and observations, adjusting the state of atmospheric constituents like CO2 to reflect observed values, thus gap-filling observations when and where they are unavailable based on previous observations and short transport simulations by GEOS. Compared to other methods, data assimilation has the advantage that it makes estimates based on our collective scientific understanding, notably of the Earth’s carbon cycle and atmospheric transport. OCO-2 GEOS (Goddard Earth Observing System) Level 3 data are produced by ingesting OCO-2 L2 retrievals every 6 hours with GEOS CoDAS, a modeling and data assimilation system maintained by NASA’s Global Modeling and Assimilation Office (GMAO). GEOS CoDAS uses a high-performance computing implementation of the Gridpoint Statistical Interpolation approach for solving the state estimation problem. GSI finds the analyzed state that minimizes the three-dimensional variational (3D-Var) cost function formulation of the state estimation problem." - zoomExtent: - - 0 - - 20 - sourceParams: - resampling_method: bilinear - variable: XCO2 - colormap_name: rdylbu_r - rescale: 0.00039394,0.000420 - legend: - unit: - label: - type: gradient - min: "< 394 ppm" - max: "> 420 ppm" - stops: - - "#4575b4" - - "#91bfdb" - - "#e0f3f8" - - "#ffffbf" - - "#fee090" - - "#fc8d59" - - "#d73027" ---- - From 4c6348f34a5b68d87419c3da8fd7522cf2a18a3e Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Fri, 10 Nov 2023 16:42:16 -0500 Subject: [PATCH 06/18] Set up an example with different datasets compare --- .../components/common/mapbox/layers/raster-timeseries.tsx | 6 ------ mock/datasets/data-from-ghg.data.mdx | 4 ++-- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/app/scripts/components/common/mapbox/layers/raster-timeseries.tsx b/app/scripts/components/common/mapbox/layers/raster-timeseries.tsx index 38a18a39d..548f54b03 100644 --- a/app/scripts/components/common/mapbox/layers/raster-timeseries.tsx +++ b/app/scripts/components/common/mapbox/layers/raster-timeseries.tsx @@ -181,7 +181,6 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { payload, controller }); - console.log(responseData); /* eslint-disable no-console */ LOG && @@ -351,14 +350,9 @@ export function MapLayerRasterTimeseries(props: MapLayerRasterTimeseriesProps) { useEffect( () => { const controller = new AbortController(); - console.log('id'); - console.log(id); async function run() { - console.log('id'); - console.log(id); let layers: AnyLayer[] = []; let sources: Record = {}; - console.log(mosaicUrl); if (mosaicUrl) { const tileParams = qs.stringify( { diff --git a/mock/datasets/data-from-ghg.data.mdx b/mock/datasets/data-from-ghg.data.mdx index 2d266517f..32c1e1e17 100644 --- a/mock/datasets/data-from-ghg.data.mdx +++ b/mock/datasets/data-from-ghg.data.mdx @@ -55,8 +55,8 @@ layers: - 0 - 0.3 compare: - datasetId: casagfed-carbonflux-monthgrid-v3 - layerId: casa-gfed-co2-flux + datasetId: nighttime-lights + layerId: nightlights-hd-monthly mapLabel: | ::js ({ dateFns, datetime, compareDatetime }) => { if (dateFns && datetime && compareDatetime) return `${dateFns.format(datetime, 'LLL yyyy')} VS ${dateFns.format(compareDatetime, 'LLL yyyy')}`; From bebe36e11c4c4082e42867d8d4b0ae6931f6cfcd Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Tue, 14 Nov 2023 21:13:27 -0500 Subject: [PATCH 07/18] Fix typo, var name bug --- .../define/use-stac-collection-search.ts | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/app/scripts/components/analysis/define/use-stac-collection-search.ts b/app/scripts/components/analysis/define/use-stac-collection-search.ts index b77188122..eeabdd789 100644 --- a/app/scripts/components/analysis/define/use-stac-collection-search.ts +++ b/app/scripts/components/analysis/define/use-stac-collection-search.ts @@ -23,6 +23,8 @@ interface UseStacSearchProps { export type DatasetWithTimeseriesData = TimeseriesDataResult & DatasetLayer & { numberOfItems: number }; +const collectionEndpointSuffix = '/collections'; + export function useStacCollectionSearch({ start, end, @@ -35,23 +37,25 @@ export function useStacCollectionSearch({ queryFn: async ({ signal }) => { const collectionUrlsFromDataSets = allAvailableDatasetsLayers .filter((dataset) => dataset.stacApiEndpoint) - .map((dataset) => `${dataset.stacApiEndpoint}/collections`) + .map( + (dataset) => `${dataset.stacApiEndpoint}${collectionEndpointSuffix}` + ) .filter((value, index, array) => array.indexOf(value) === index); const collectionUrls = [ ...collectionUrlsFromDataSets, - `${process.env.API_STAC_ENDPOINT}/collections` + `${process.env.API_STAC_ENDPOINT}${collectionEndpointSuffix}` ]; - const colloectionRequests = collectionUrls.map((url: string) => + const collectionRequests = collectionUrls.map((url: string) => axios.get(url, { signal }).then((response) => { return response.data.collections.map((col) => ({ ...col, - stacApiEndpoint: url.replace(/\/collections$/, '') + stacApiEndpoint: url.replace(collectionEndpointSuffix, '') })); }) ); - return axios.all(colloectionRequests).then( + return axios.all(collectionRequests).then( axios.spread((...responses) => { // Merge all responses into one array const mergedData = [].concat(...responses); @@ -103,14 +107,14 @@ export function useStacCollectionSearch({ function getInTemporalAndSpatialExtent(collectionData, aoi, timeRange) { const matchingCollectionIds = collectionData.reduce((acc, col) => { - const { id, stacEndpoint } = col; + const { id, stacApiEndpoint } = col; // Is is a dataset defined in the app? // If not, skip other calculations. const isAppDataset = allAvailableDatasetsLayers.some((l) => { const stacApiEndpointUsed = l.stacApiEndpoint ?? process.env.API_STAC_ENDPOINT; - return l.stacCol === id && stacApiEndpointUsed === stacEndpoint; + return l.stacCol === id && stacApiEndpointUsed === stacApiEndpoint; }); if ( From db7a4678d2ca89e57f6ac1f6d2412e19ee327dbf Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 14:31:28 +0000 Subject: [PATCH 08/18] Add tour step for compare --- .../components/timeline/timeline-controls.tsx | 8 ++++++- .../components/exploration/tour-manager.tsx | 22 ++++++++++++++----- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/app/scripts/components/exploration/components/timeline/timeline-controls.tsx b/app/scripts/components/exploration/components/timeline/timeline-controls.tsx index db3130158..0cea0fe60 100644 --- a/app/scripts/components/exploration/components/timeline/timeline-controls.tsx +++ b/app/scripts/components/exploration/components/timeline/timeline-controls.tsx @@ -96,7 +96,12 @@ export function TimelineControls(props: TimelineControlsProps) { setSelectedDay(d.start!); }} renderTriggerElement={(props, label) => ( - + A {label} @@ -139,6 +144,7 @@ export function TimelineControls(props: TimelineControlsProps) { ) : ( { if (!xScaled || !selectedDay) return; const [, max] = xScaled.range(); diff --git a/app/scripts/components/exploration/tour-manager.tsx b/app/scripts/components/exploration/tour-manager.tsx index a90858357..ff88f90e5 100644 --- a/app/scripts/components/exploration/tour-manager.tsx +++ b/app/scripts/components/exploration/tour-manager.tsx @@ -61,6 +61,17 @@ const introTourSteps = [ selector: "[data-tour='date-picker-a']", content: 'Alternatively you can also select a date through the date picker.' }, + { + title: 'Compare dates', + selector: "[data-tour='compare-date']", + content: () => ( + <> + You can also select a second date to compare with the first one. +
+ The map will show a slider allowing you to view the differences. + + ) + }, { title: 'Timeline', selector: "[data-tour='timeline-interaction-rect']", @@ -92,8 +103,8 @@ const analysisTourSteps = [ selector: "[data-tour='analysis-message']", content: () => ( <> - You can now calculate a time series of zonal - statistics for your area of interest. + You can now calculate a time series of zonal statistics for your area of + interest. ), stepInteraction: false @@ -103,10 +114,11 @@ const analysisTourSteps = [ selector: "[data-tour='analysis-toolbar']", content: () => ( <> - Refine the date range to analyze with the data pickers - or handles on the timeline. + Refine the date range to analyze with the data pickers or handles on the + timeline.
- Once you're happy, press the analyze button to start the calculation. + Once you're happy, press the analyze button to start the + calculation. ), stepInteraction: false From b2bc5dc5bdfdc92a0117ff11a74fa678c0c09f39 Mon Sep 17 00:00:00 2001 From: Hanbyul Jo Date: Tue, 14 Nov 2023 21:28:01 -0500 Subject: [PATCH 09/18] Use set to get unique values --- .../define/use-stac-collection-search.ts | 26 ++++++++----------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/app/scripts/components/analysis/define/use-stac-collection-search.ts b/app/scripts/components/analysis/define/use-stac-collection-search.ts index eeabdd789..892f08030 100644 --- a/app/scripts/components/analysis/define/use-stac-collection-search.ts +++ b/app/scripts/components/analysis/define/use-stac-collection-search.ts @@ -35,17 +35,16 @@ export function useStacCollectionSearch({ const result = useQuery({ queryKey: ['stacCollection'], queryFn: async ({ signal }) => { + const collectionUrlsFromDataSets = allAvailableDatasetsLayers - .filter((dataset) => dataset.stacApiEndpoint) - .map( - (dataset) => `${dataset.stacApiEndpoint}${collectionEndpointSuffix}` - ) - .filter((value, index, array) => array.indexOf(value) === index); - - const collectionUrls = [ - ...collectionUrlsFromDataSets, - `${process.env.API_STAC_ENDPOINT}${collectionEndpointSuffix}` - ]; + .filter((dataset) => dataset.stacApiEndpoint) + .map( + (dataset) => + `${dataset.stacApiEndpoint}${collectionEndpointSuffix}` + ); + // Get unique values of stac api endpoints from layer data, concat with api_stac_endpoiint from env var + const collectionUrls = Array.from(new Set(collectionUrlsFromDataSets)) + .concat(`${process.env.API_STAC_ENDPOINT}${collectionEndpointSuffix}`); const collectionRequests = collectionUrls.map((url: string) => axios.get(url, { signal }).then((response) => { @@ -56,11 +55,8 @@ export function useStacCollectionSearch({ }) ); return axios.all(collectionRequests).then( - axios.spread((...responses) => { - // Merge all responses into one array - const mergedData = [].concat(...responses); - return mergedData; - }) + // Merge all responses into one array + axios.spread((...responses) => [].concat(...responses)) ); }, enabled: readyToLoadDatasets From dccf06efd00c7fd3d7f9b81268f97e048c913728 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 15:21:19 +0000 Subject: [PATCH 10/18] Connect data catalog to exploration --- app/scripts/components/data-catalog/index.tsx | 1 + .../exploration/components/dataset-selector-modal.tsx | 5 +++++ app/scripts/utils/routes.ts | 7 +++++-- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/app/scripts/components/data-catalog/index.tsx b/app/scripts/components/data-catalog/index.tsx index 7af9992fe..3f8e7ca1d 100644 --- a/app/scripts/components/data-catalog/index.tsx +++ b/app/scripts/components/data-catalog/index.tsx @@ -89,6 +89,7 @@ export const prepareDatasets = ( d.name.toLowerCase().includes(searchLower) || d.description.toLowerCase().includes(searchLower) || d.layers.some((l) => l.stacCol.toLowerCase().includes(searchLower)) || + d.layers.some((l) => l.name.toLowerCase().includes(searchLower)) || topicsTaxonomy?.values.some((t) => t.name.toLowerCase().includes(searchLower) ) diff --git a/app/scripts/components/exploration/components/dataset-selector-modal.tsx b/app/scripts/components/exploration/components/dataset-selector-modal.tsx index cb0b57b22..8088698a5 100644 --- a/app/scripts/components/exploration/components/dataset-selector-modal.tsx +++ b/app/scripts/components/exploration/components/dataset-selector-modal.tsx @@ -214,8 +214,13 @@ export function DatasetSelectorModal(props: DatasetSelectorModalProps) { const search = controlVars.search ?? ''; // Clear filters when the modal is revealed. + const firstRevealRef = React.useRef(true); useEffect(() => { if (revealed) { + if (firstRevealRef.current) { + firstRevealRef.current = false; + return; + } onAction(Actions.CLEAR); } }, [revealed]); diff --git a/app/scripts/utils/routes.ts b/app/scripts/utils/routes.ts index 345182e8b..9fbbb0998 100644 --- a/app/scripts/utils/routes.ts +++ b/app/scripts/utils/routes.ts @@ -5,6 +5,7 @@ export const STORIES_PATH = '/stories'; export const DATASETS_PATH = '/data-catalog'; export const ANALYSIS_PATH = '/analysis'; export const ANALYSIS_RESULTS_PATH = '/analysis/results'; +export const EXPLORATION_PATH = '/exploration'; export const getStoryPath = (d: StoryData | string) => `${STORIES_PATH}/${typeof d === 'string' ? d : d.id}`; @@ -12,5 +13,7 @@ export const getStoryPath = (d: StoryData | string) => export const getDatasetPath = (d: DatasetData | string) => `${DATASETS_PATH}/${typeof d === 'string' ? d : d.id}`; -export const getDatasetExplorePath = (d: DatasetData | string) => - `${DATASETS_PATH}/${typeof d === 'string' ? d : d.id}/explore`; +export const getDatasetExplorePath = (d: DatasetData | string) => { + const id = typeof d === 'string' ? d : d.id; + return `${EXPLORATION_PATH}?search=${id}`; +}; From e21789df19affc14b6493236a17a2b1b0850b24c Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 15:21:35 +0000 Subject: [PATCH 11/18] Add text search highlight to dataset selector modal --- .../components/dataset-selector-modal.tsx | 29 ++++++++++++++++--- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/app/scripts/components/exploration/components/dataset-selector-modal.tsx b/app/scripts/components/exploration/components/dataset-selector-modal.tsx index 8088698a5..d471f5f94 100644 --- a/app/scripts/components/exploration/components/dataset-selector-modal.tsx +++ b/app/scripts/components/exploration/components/dataset-selector-modal.tsx @@ -51,6 +51,7 @@ import { } from '$components/common/browse-controls/use-browse-controls'; import { prepareDatasets, sortOptions } from '$components/data-catalog'; import Pluralize from '$utils/pluralize'; +import TextHighlight from '$components/common/text-highlight'; const DatasetModal = styled(Modal)` z-index: ${themeVal('zIndices.modal')}; @@ -304,6 +305,7 @@ export function DatasetSelectorModal(props: DatasetSelectorModalProps) { return (
  • void; } function DatasetLayerCard(props: DatasetLayerProps) { - const { parent, layer, onDatasetClick, selected } = props; + const { parent, layer, onDatasetClick, selected, searchTerm } = props; const topics = getTaxonomy(parent, TAXONOMY_TOPICS)?.values; @@ -376,8 +379,19 @@ function DatasetLayerCard(props: DatasetLayerProps) { e.preventDefault(); onDatasetClick(); }} - title={layer.name} - description={`From: ${parent.name}`} + title={ + + {layer.name} + + } + description={ + <> + From:{' '} + + {parent.name} + + + } imgSrc={parent.media?.src} imgAlt={parent.media?.alt} footerContent={ @@ -387,7 +401,14 @@ function DatasetLayerCard(props: DatasetLayerProps) {
    Topics
    {topics.map((t) => (
    - {t.name} + + + {t.name} + +
    ))} From fc8a0a72bdb11a49d51903bf1029d8fc9cdec0cd Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 15:32:06 +0000 Subject: [PATCH 12/18] Remove analysis from menu --- app/scripts/components/common/page-header.tsx | 8 ++++---- app/scripts/main.tsx | 15 --------------- app/scripts/utils/routes.ts | 2 -- 3 files changed, 4 insertions(+), 21 deletions(-) diff --git a/app/scripts/components/common/page-header.tsx b/app/scripts/components/common/page-header.tsx index fbf57779e..48e9b4629 100644 --- a/app/scripts/components/common/page-header.tsx +++ b/app/scripts/components/common/page-header.tsx @@ -30,8 +30,8 @@ import { variableGlsp } from '$styles/variable-utils'; import { STORIES_PATH, DATASETS_PATH, - ANALYSIS_PATH, - ABOUT_PATH + ABOUT_PATH, + EXPLORATION_PATH } from '$utils/routes'; import GlobalMenuLinkCSS from '$styles/menu-link'; import { useMediaQuery } from '$utils/use-media-query'; @@ -413,10 +413,10 @@ function PageHeader() {
  • - Data Analysis + Exploration
  • diff --git a/app/scripts/main.tsx b/app/scripts/main.tsx index f5f28b477..c3d5153cb 100644 --- a/app/scripts/main.tsx +++ b/app/scripts/main.tsx @@ -26,12 +26,8 @@ const StoriesHub = lazy(() => import('$components/stories/hub')); const StoriesSingle = lazy(() => import('$components/stories/single')); const DataCatalog = lazy(() => import('$components/data-catalog')); -const DatasetsExplore = lazy(() => import('$components/datasets/s-explore')); const DatasetsOverview = lazy(() => import('$components/datasets/s-overview')); -const Analysis = lazy(() => import('$components/analysis/define')); -const AnalysisResults = lazy(() => import('$components/analysis/results')); - const Exploration = lazy(() => import('$components/exploration')); const Sandbox = lazy(() => import('$components/sandbox')); @@ -45,8 +41,6 @@ const DevseedUiThemeProvider = DsTp as any; import { ReactQueryProvider } from '$context/react-query'; import { ABOUT_PATH, - ANALYSIS_PATH, - ANALYSIS_RESULTS_PATH, DATASETS_PATH, STORIES_PATH } from '$utils/routes'; @@ -93,20 +87,11 @@ function Root() { path={`${DATASETS_PATH}/:datasetId`} element={} /> - } - /> } /> } /> - } /> - } - /> } /> } /> diff --git a/app/scripts/utils/routes.ts b/app/scripts/utils/routes.ts index 9fbbb0998..76dc9281d 100644 --- a/app/scripts/utils/routes.ts +++ b/app/scripts/utils/routes.ts @@ -3,8 +3,6 @@ import { DatasetData, StoryData } from 'veda'; export const ABOUT_PATH = '/about'; export const STORIES_PATH = '/stories'; export const DATASETS_PATH = '/data-catalog'; -export const ANALYSIS_PATH = '/analysis'; -export const ANALYSIS_RESULTS_PATH = '/analysis/results'; export const EXPLORATION_PATH = '/exploration'; export const getStoryPath = (d: StoryData | string) => From 68d327b893abbbd956598cff7114a40864e57e00 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 15:43:05 +0000 Subject: [PATCH 13/18] Add info icon to dataset card Fix #747 --- .../components/datasets/dataset-list-item.tsx | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx index c2abe2a19..43b2bd52c 100644 --- a/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx +++ b/app/scripts/components/exploration/components/datasets/dataset-list-item.tsx @@ -1,4 +1,5 @@ import React, { useCallback, useMemo } from 'react'; +import { Link } from 'react-router-dom'; import { useAtomValue } from 'jotai'; import { Reorder, useDragControls } from 'framer-motion'; import styled, { useTheme } from 'styled-components'; @@ -6,6 +7,7 @@ import { addDays, subDays, areIntervalsOverlapping } from 'date-fns'; import { useQueryClient } from '@tanstack/react-query'; import { ScaleTime } from 'd3'; import { + CollecticonCircleInformation, CollecticonEye, CollecticonEyeDisabled, CollecticonGripVertical @@ -13,6 +15,7 @@ import { import { glsp, themeVal } from '@devseed-ui/theme-provider'; import { Toolbar, ToolbarIconButton } from '@devseed-ui/toolbar'; import { Heading } from '@devseed-ui/typography'; +import { Button } from '@devseed-ui/button'; import { DatasetPopover, @@ -49,6 +52,8 @@ import { useAnalysisController, useAnalysisDataRequest } from '$components/exploration/hooks/use-analysis-data-request'; +import { getDatasetPath } from '$utils/routes'; +import { findParentDataset } from '$components/exploration/data-utils'; const DatasetItem = styled.article` width: 100%; @@ -223,6 +228,17 @@ export function DatasetListItem(props: DatasetListItemProps) { {dataset.data.name} + setVisible((v) => !v)}> {isVisible ? ( From f4845cc4d3437c736d23caff33b96bc144774edb Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 16:14:00 +0000 Subject: [PATCH 14/18] Implement custom endpoint in A&E page --- .../style-generators/raster-timeseries.tsx | 15 ++++++++++--- .../components/exploration/analysis-data.ts | 21 ++++++++++++++++--- .../exploration/components/map/layer.tsx | 2 ++ .../hooks/use-stac-metadata-datasets.ts | 6 ++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/app/scripts/components/common/map/style-generators/raster-timeseries.tsx b/app/scripts/components/common/map/style-generators/raster-timeseries.tsx index 9f2bbac5c..e3ed21a28 100644 --- a/app/scripts/components/common/map/style-generators/raster-timeseries.tsx +++ b/app/scripts/components/common/map/style-generators/raster-timeseries.tsx @@ -45,6 +45,8 @@ export interface RasterTimeseriesProps extends BaseGeneratorParams { bounds?: number[]; onStatusChange?: (result: { status: ActionStatus; id: string }) => void; isPositionSet?: boolean; + stacApiEndpoint?: string; + tileApiEndpoint?: string; } enum STATUS_KEY { @@ -70,7 +72,9 @@ export function RasterTimeseries(props: RasterTimeseriesProps) { onStatusChange, isPositionSet, hidden, - opacity + opacity, + stacApiEndpoint, + tileApiEndpoint } = props; const { current: mapInstance } = useMaps(); @@ -81,6 +85,11 @@ export function RasterTimeseries(props: RasterTimeseriesProps) { const minZoom = zoomExtent?.[0] ?? 0; const generatorId = `raster-timeseries-${id}`; + const stacApiEndpointToUse = + stacApiEndpoint ?? process.env.API_STAC_ENDPOINT ?? ''; + const tileApiEndpointToUse = + tileApiEndpoint ?? process.env.API_RASTER_ENDPOINT ?? ''; + // Status tracking. // A raster timeseries layer has a base layer and may have markers. // The status is succeeded only if all requests succeed. @@ -165,7 +174,7 @@ export function RasterTimeseries(props: RasterTimeseriesProps) { /* eslint-enable no-console */ const responseData = await requestQuickCache({ - url: `${process.env.API_STAC_ENDPOINT}/search`, + url: `${stacApiEndpointToUse}/search`, payload, controller }); @@ -262,7 +271,7 @@ export function RasterTimeseries(props: RasterTimeseriesProps) { /* eslint-enable no-console */ const responseData = await requestQuickCache({ - url: `${process.env.API_RASTER_ENDPOINT}/mosaic/register`, + url: `${tileApiEndpointToUse}/mosaic/register`, payload, controller }); diff --git a/app/scripts/components/exploration/analysis-data.ts b/app/scripts/components/exploration/analysis-data.ts index 66cf13f6a..70b63d1b2 100644 --- a/app/scripts/components/exploration/analysis-data.ts +++ b/app/scripts/components/exploration/analysis-data.ts @@ -16,6 +16,7 @@ import { interface DatasetAssetsRequestParams { stacCol: string; assets: string; + stacEndpoint: string; dateStart: Date; dateEnd: Date; aoi: FeatureCollection; @@ -30,11 +31,18 @@ interface DatasetAssetsRequestParams { * @returns Promise with the asset urls */ async function getDatasetAssets( - { dateStart, dateEnd, stacCol, assets, aoi }: DatasetAssetsRequestParams, + { + dateStart, + dateEnd, + stacCol, + assets, + aoi, + stacEndpoint + }: DatasetAssetsRequestParams, opts: AxiosRequestConfig ): Promise<{ assets: { date: Date; url: string }[] }> { const searchReqRes = await axios.post( - `${process.env.API_STAC_ENDPOINT}/search`, + `${stacEndpoint}/search`, { 'filter-lang': 'cql2-json', limit: 10000, @@ -96,6 +104,9 @@ export async function requestDatasetTimeseriesData({ meta: {} }); + const stacApiEndpointToUse = + datasetData.stacApiEndpoint ?? process.env.API_STAC_ENDPOINT ?? ''; + try { const layerInfoFromSTAC = await concurrencyManager.queue( `${id}-analysis`, @@ -105,6 +116,7 @@ export async function requestDatasetTimeseriesData({ ({ signal }) => getDatasetAssets( { + stacEndpoint: stacApiEndpointToUse, stacCol: datasetData.stacCol, assets: datasetData.sourceParams?.assets || 'cog_default', aoi, @@ -152,6 +164,9 @@ export async function requestDatasetTimeseriesData({ let loaded = 0; + const tileEndpointToUse = + datasetData.tileApiEndpoint ?? process.env.API_RASTER_ENDPOINT ?? ''; + const layerStatistics = await Promise.all( assets.map(async ({ date, url }) => { const statistics = await concurrencyManager.queue( @@ -161,7 +176,7 @@ export async function requestDatasetTimeseriesData({ ['analysis', id, 'asset', url, aoi], async ({ signal }) => { const { data } = await axios.post( - `${process.env.API_RASTER_ENDPOINT}/cog/statistics?url=${url}`, + `${tileEndpointToUse}/cog/statistics?url=${url}`, // Making a request with a FC causes a 500 (as of 2023/01/20) fixAoiFcForStacSearch(aoi), { signal } diff --git a/app/scripts/components/exploration/components/map/layer.tsx b/app/scripts/components/exploration/components/map/layer.tsx index f93d4286e..27d2da378 100644 --- a/app/scripts/components/exploration/components/map/layer.tsx +++ b/app/scripts/components/exploration/components/map/layer.tsx @@ -49,6 +49,8 @@ export function Layer(props: LayerProps) { { - const { type, stacCol } = dataset.data; + const { type, stacCol, stacApiEndpoint } = dataset.data; + + const stacApiEndpointToUse = stacApiEndpoint ?? process.env.API_STAC_ENDPOINT; const { data } = await axios.get( - `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}` + `${stacApiEndpointToUse}/collections/${stacCol}` ); const commonTimeseriesParams = { From 93b3d067ace74dc421a4740c32ff50a20c90b060 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 16:33:40 +0000 Subject: [PATCH 15/18] Ensure that range is within domain when updated --- .../components/timeline/timeline.tsx | 41 ++++++++++++++----- 1 file changed, 30 insertions(+), 11 deletions(-) diff --git a/app/scripts/components/exploration/components/timeline/timeline.tsx b/app/scripts/components/exploration/components/timeline/timeline.tsx index 383fd24a2..4521b9924 100644 --- a/app/scripts/components/exploration/components/timeline/timeline.tsx +++ b/app/scripts/components/exploration/components/timeline/timeline.tsx @@ -137,6 +137,18 @@ interface TimelineProps { onDatasetAddClick: () => void; } +const getIntervalFromDate = (selectedDay: Date, dataDomain: [Date, Date]) => { + const startDate = sub(selectedDay, { months: 2 }); + const endDate = add(selectedDay, { months: 2 }); + + // Set start and end days from the selected day, if able. + const [start, end] = dataDomain; + return { + start: isAfter(startDate, start) ? startDate : selectedDay, + end: isBefore(endDate, end) ? endDate : end + }; +}; + export default function Timeline(props: TimelineProps) { const { onDatasetAddClick } = props; @@ -318,7 +330,8 @@ export default function Timeline(props: TimelineProps) { ); const successDatasets = datasets.filter( - (d): d is TimelineDatasetSuccess => d.status === TimelineDatasetStatus.SUCCESS + (d): d is TimelineDatasetSuccess => + d.status === TimelineDatasetStatus.SUCCESS ); // When a loaded dataset is added from an empty state, compute the correct @@ -353,9 +366,23 @@ export default function Timeline(props: TimelineProps) { // available dataset date. We can't use the date domain, because the end of // the domain is the max date + a duration so that all dataset dates fit in // the timeline. + let newSelectedDay; // needed for the interval if (!selectedDay || !isWithinInterval(selectedDay, { start, end })) { - const maxDate = max(successDatasets.map(d => d.data.domain.last)); + const maxDate = max(successDatasets.map((d) => d.data.domain.last)); setSelectedDay(maxDate); + newSelectedDay = maxDate; + } else { + newSelectedDay = selectedDay; + } + + // If there is a selected interval and is not within the new domain, + // calculate a new one. + if ( + selectedInterval && + (!isWithinInterval(selectedInterval.start, { start, end }) || + !isWithinInterval(selectedInterval.end, { start, end })) + ) { + setSelectedInterval(getIntervalFromDate(newSelectedDay, dataDomain)); } }, [ prevDataDomain, @@ -381,15 +408,7 @@ export default function Timeline(props: TimelineProps) { if (prevFeaturesCount === 0 && features.length > 0) { // We went from 0 features to some features. - const startDate = sub(selectedDay, { months: 2 }); - const endDate = add(selectedDay, { months: 2 }); - - // Set start and end days from the selected day, if able. - const [start, end] = dataDomain; - setSelectedInterval({ - start: isAfter(startDate, start) ? startDate : selectedDay, - end: isBefore(endDate, end) ? endDate : end - }); + setSelectedInterval(getIntervalFromDate(selectedDay, dataDomain)); } }, [ features.length, From 817d5964ff3a57103af511d86681dcdab9f0ffb4 Mon Sep 17 00:00:00 2001 From: Daniel da Silva Date: Wed, 15 Nov 2023 17:12:16 +0000 Subject: [PATCH 16/18] Add support for vector datasets on exploration page --- .../style-generators/vector-timeseries.tsx | 316 ++++++++++++++++++ .../components/dataset-selector-modal.tsx | 2 +- .../exploration/components/map/layer.tsx | 47 ++- .../components/exploration/data-utils.ts | 2 +- 4 files changed, 350 insertions(+), 17 deletions(-) create mode 100644 app/scripts/components/common/map/style-generators/vector-timeseries.tsx diff --git a/app/scripts/components/common/map/style-generators/vector-timeseries.tsx b/app/scripts/components/common/map/style-generators/vector-timeseries.tsx new file mode 100644 index 000000000..74ff9efc5 --- /dev/null +++ b/app/scripts/components/common/map/style-generators/vector-timeseries.tsx @@ -0,0 +1,316 @@ +import { useCallback, useEffect, useMemo, useState } from 'react'; +import qs from 'qs'; +import { + AnyLayer, + AnySourceImpl, + VectorSourceImpl +} from 'mapbox-gl'; +import { useTheme } from 'styled-components'; +import { endOfDay, startOfDay } from 'date-fns'; +import centroid from '@turf/centroid'; +import { LngLatLike } from 'react-map-gl'; +import { Feature } from 'geojson'; + +import { BaseGeneratorParams } from '../types'; +import useMapStyle from '../hooks/use-map-style'; +import { + requestQuickCache +} from '../utils'; +import useFitBbox from '../hooks/use-fit-bbox'; +import useLayerInteraction from '../hooks/use-layer-interaction'; +import { MARKER_LAYOUT } from '../hooks/use-custom-marker'; +import useMaps from '../hooks/use-maps'; +import useGeneratorParams from '../hooks/use-generator-params'; + +import { + ActionStatus, + S_FAILED, + S_LOADING, + S_SUCCEEDED +} from '$utils/status'; +import { userTzDate2utcString } from '$utils/date'; + +export interface VectorTimeseriesProps extends BaseGeneratorParams { + id: string; + stacCol: string; + date: Date; + sourceParams?: Record; + zoomExtent?: number[]; + bounds?: number[]; + onStatusChange?: (result: { status: ActionStatus; id: string }) => void; + isPositionSet?: boolean; + stacApiEndpoint?: string; +} + + +export function VectorTimeseries(props: VectorTimeseriesProps) { + const { + id, + stacCol, + date, + sourceParams, + zoomExtent, + bounds, + onStatusChange, + isPositionSet, + hidden, + opacity, + stacApiEndpoint + } = props; + + const { current: mapInstance } = useMaps(); + + const theme = useTheme(); + const { updateStyle } = useMapStyle(); + const [featuresApiEndpoint, setFeaturesApiEndpoint] = useState(''); + const [featuresBbox, setFeaturesBbox] = + useState<[number, number, number, number]>(); + + const [minZoom, maxZoom] = zoomExtent ?? [0, 20]; + const generatorId = `vector-timeseries-${id}`; + + const stacApiEndpointToUse = + stacApiEndpoint ?? process.env.API_STAC_ENDPOINT ?? ''; + + // + // Get the tiles url + // + useEffect(() => { + const controller = new AbortController(); + + async function load() { + try { + onStatusChange?.({ status: S_LOADING, id }); + const data = await requestQuickCache({ + url: `${stacApiEndpointToUse}/collections/${stacCol}`, + method: 'GET', + controller + }); + + const endpoint = data.links.find((l) => l.rel === 'external').href; + setFeaturesApiEndpoint(endpoint); + + const featuresData = await requestQuickCache({ + url: endpoint, + method: 'GET', + controller + }); + + if (featuresData.extent.spatial.bbox) { + setFeaturesBbox(featuresData.extent.spatial.bbox[0]); + } + + onStatusChange?.({ status: S_SUCCEEDED, id }); + } catch (error) { + if (!controller.signal.aborted) { + setFeaturesApiEndpoint(''); + onStatusChange?.({ status: S_FAILED, id }); + } + return; + } + } + + load(); + + return () => { + controller.abort(); + }; + }, [mapInstance, id, stacCol, stacApiEndpointToUse, date, onStatusChange]); + + + // + // Generate Mapbox GL layers and sources for vector timeseries + // + const haveSourceParamsChanged = useMemo( + () => JSON.stringify(sourceParams), + [sourceParams] + ); + + const generatorParams = useGeneratorParams(props); + + useEffect(() => { + if (!date || !featuresApiEndpoint) return; + + const start = userTzDate2utcString(startOfDay(date)); + const end = userTzDate2utcString(endOfDay(date)); + + const tileParams = qs.stringify({ + ...sourceParams, + datetime: `${start}/${end}` + }); + + const vectorOpacity = typeof opacity === 'number' ? opacity / 100 : 1; + + const sources: Record = { + [id]: { + type: 'vector', + tiles: [`${featuresApiEndpoint}/tiles/{z}/{x}/{y}?${tileParams}`] + } as VectorSourceImpl + }; + + const layers = [ + { + id: `${id}-line-bg`, + type: 'line', + source: id, + 'source-layer': 'default', + paint: { + 'line-opacity': hidden ? 0 : vectorOpacity, + 'line-opacity-transition': { + duration: 320 + }, + 'line-color': theme.color?.['danger-300'], + 'line-width': [ + 'interpolate', + ['linear'], + ['zoom'], + minZoom, + 4, + maxZoom, + 10 + ] + }, + // filter: ['==', '$type', 'LineString'], + minzoom: minZoom, + metadata: { + id, + layerOrderPosition: 'raster', + xyzTileUrl: `${featuresApiEndpoint}/tiles/{z}/{x}/{y}?${tileParams}` + } + }, + { + id: `${id}-line-fg`, + type: 'line', + source: id, + 'source-layer': 'default', + paint: { + 'line-opacity': hidden ? 0 : vectorOpacity, + 'line-opacity-transition': { + duration: 320 + }, + 'line-color': theme.color?.infographicB, + 'line-width': [ + 'interpolate', + ['linear'], + ['zoom'], + minZoom, + 2, + maxZoom, + 5 + ] + }, + filter: ['==', '$type', 'LineString'], + minzoom: minZoom, + metadata: { + layerOrderPosition: 'raster' + } + }, + { + id: `${id}-fill-fg`, + type: 'fill', + source: id, + 'source-layer': 'default', + paint: { + 'fill-opacity': hidden ? 0 : Math.min(vectorOpacity, 0.8), + 'fill-opacity-transition': { + duration: 320 + }, + 'fill-color': theme.color?.infographicB, + }, + filter: ['==', '$type', 'Polygon'], + minzoom: minZoom, + metadata: { + layerOrderPosition: 'raster' + } + }, + minZoom > 0 + ? { + type: 'symbol', + id: `${id}-points`, + source: id, + 'source-layer': 'default', + layout: { + ...(MARKER_LAYOUT as any), + visibility: hidden ? 'none' : 'visible' + }, + paint: { + 'icon-color': theme.color?.infographicB, + 'icon-halo-color': theme.color?.base, + 'icon-halo-width': 1 + }, + maxzoom: minZoom, + metadata: { + layerOrderPosition: 'markers' + } + } + : undefined + ].filter(Boolean) as AnyLayer[]; + + updateStyle({ + generatorId, + sources, + layers, + params: generatorParams + }); + // sourceParams not included, but using a stringified version of it to + // detect changes (haveSourceParamsChanged) + // `theme` will not change throughout the app use + }, [ + id, + updateStyle, + date, + featuresApiEndpoint, + minZoom, + maxZoom, + hidden, + opacity, + generatorParams, + haveSourceParamsChanged, + generatorId + ]); + + // + // Cleanup layers on unmount. + // + useEffect(() => { + return () => { + updateStyle({ + generatorId, + sources: {}, + layers: [] + }); + }; + }, [updateStyle, generatorId]); + + // + // Listen to mouse events on the markers layer + // + const onPointsClick = useCallback( + (features) => { + const extractedFeat = { + type: 'Feature', + geometry: features[0].geometry + } as Feature; + + const center = centroid(extractedFeat).geometry.coordinates as LngLatLike; + + // Zoom past the min zoom centering on the clicked feature. + mapInstance?.flyTo({ + zoom: minZoom, + center + }); + }, + [mapInstance, minZoom] + ); + useLayerInteraction({ + layerId: `${id}-points`, + onClick: onPointsClick + }); + + // + // FitBounds when needed + // + useFitBbox(!!isPositionSet, bounds, featuresBbox); + + return null; +} diff --git a/app/scripts/components/exploration/components/dataset-selector-modal.tsx b/app/scripts/components/exploration/components/dataset-selector-modal.tsx index d471f5f94..853993c2c 100644 --- a/app/scripts/components/exploration/components/dataset-selector-modal.tsx +++ b/app/scripts/components/exploration/components/dataset-selector-modal.tsx @@ -238,7 +238,7 @@ export function DatasetSelectorModal(props: DatasetSelectorModalProps) { sortDir }) .flatMap((dataset) => dataset.layers) - .filter((d) => d.type !== 'vector' && !d.analysis?.exclude), + .filter((d) => !d.analysis?.exclude), [search, taxonomies, sortField, sortDir] ); diff --git a/app/scripts/components/exploration/components/map/layer.tsx b/app/scripts/components/exploration/components/map/layer.tsx index 27d2da378..a10f64068 100644 --- a/app/scripts/components/exploration/components/map/layer.tsx +++ b/app/scripts/components/exploration/components/map/layer.tsx @@ -9,8 +9,9 @@ import { useTimelineDatasetSettings } from '../../atoms/hooks'; -import { RasterTimeseries } from '$components/common/map/style-generators/raster-timeseries'; import { resolveConfigFunctions } from '$components/common/map/utils'; +import { RasterTimeseries } from '$components/common/map/style-generators/raster-timeseries'; +import { VectorTimeseries } from '$components/common/map/style-generators/vector-timeseries'; interface LayerProps { id: string; @@ -45,18 +46,34 @@ export function Layer(props: LayerProps) { return resolveConfigFunctions(dataset.data, bag); }, [dataset, relevantDate]); - return ( -