From 3ec815504188723754a402592b374e296a609cc1 Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Thu, 18 Jul 2024 13:08:10 +0200 Subject: [PATCH 01/62] added grib_parser --- requirements/dev-environment.yml | 1 + requirements/dev-requirements.yml | 1 + requirements/environment.yml | 1 + requirements/requirements.yml | 1 + templates/ICON.jinja | 1 + util/model_output_parser.py | 76 +++++++++++++++++++++++++++++++ 6 files changed, 81 insertions(+) diff --git a/requirements/dev-environment.yml b/requirements/dev-environment.yml index 55935c91..6ac4dddb 100644 --- a/requirements/dev-environment.yml +++ b/requirements/dev-environment.yml @@ -255,5 +255,6 @@ dependencies: - zipp=3.11.0 - zlib=1.2.13 - zstd=1.5.2 + - earthkit-data - pip: - flake8-pyproject==1.2.2 diff --git a/requirements/dev-requirements.yml b/requirements/dev-requirements.yml index 1be4950f..e9f7a4a1 100644 --- a/requirements/dev-requirements.yml +++ b/requirements/dev-requirements.yml @@ -22,5 +22,6 @@ dependencies: - sphinx-autobuild - toml>=0.10.2 - pip + - earthkit-data - pip: - flake8-pyproject diff --git a/requirements/environment.yml b/requirements/environment.yml index b5927ea4..77c37e49 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -175,3 +175,4 @@ dependencies: - xz=5.2.6 - zlib=1.2.13 - zstd=1.5.2 + - earthkit-data diff --git a/requirements/requirements.yml b/requirements/requirements.yml index 5d12bc51..01b8ec9d 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -15,5 +15,6 @@ dependencies: - scipy>=1.4.1 - xarray>=0.16.1 - pip + - earthkit-data - pip: # add pip requirements here if needed. diff --git a/templates/ICON.jinja b/templates/ICON.jinja index 3144218b..54ce9522 100644 --- a/templates/ICON.jinja +++ b/templates/ICON.jinja @@ -10,6 +10,7 @@ "member_num": "[{member_num}]", "member_type": "{{member_type}}", "file_specification": [{ + "GRIB": {"format": "GRIB", "time_dim": "step", "horizontal_dims": ["values"], "var_excl": ["tlon", "tlat", "vlon", "vlat", "ulon", "ulat", "h", "slor", "anor", "isor", "sdor"]}, "latlon": { "format": "netcdf", "time_dim": "time", "horizontal_dims": ["lat:lon"] }, "meteogram": { "format": "netcdf", "time_dim": "time", "horizontal_dims": ["max_nlevs:nstations", "nstations"] }, "dace":{ "format": "netcdf", "time_dim": null, "horizontal_dims": ["d_body"]}, diff --git a/util/model_output_parser.py b/util/model_output_parser.py index bd550f79..5aa253ee 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -4,6 +4,7 @@ import numpy as np import pandas as pd import xarray +import earthkit.data from util.constants import compute_statistics from util.log_handler import logger @@ -60,6 +61,80 @@ def parse_netcdf(file_id, filename, specification): ds.close() return var_dfs +def parse_grib(file_id, filename, specification): + logger.debug("parse GRIB file {}".format(filename)) + time_dim = specification["time_dim"] + horizontal_dims = specification["horizontal_dims"] + fill_value_key = specification.get("fill_value_key", None) + + ds_grib = earthkit.data.from_source("file", filename) + short_name_excl = specification["var_excl"] + + short_name = np.unique(ds_grib.metadata("shortName")) + short_name = short_name[np.where(~np.isin(short_name, short_name_excl))].tolist() + + level_type = np.unique(ds_grib.metadata("typeOfLevel")).tolist() + level_type.remove("unknown") + + + var_dfs = [] + for lev in level_type: + paramId = np.unique(ds_grib.sel(typeOfLevel=lev, shortName=short_name).metadata("paramId")).tolist() + for pid in paramId: + ds_temp = get_ds(ds_grib, pid, lev) + v = list(ds_temp.keys())[0] + + dim_to_squeeze = [dim for dim, size in zip(ds_temp[v].dims, ds_temp[v].shape) if size==1 and dim != time_dim] + ds = ds_temp.squeeze(dim=dim_to_squeeze) + + sub_df = dataframe_from_ncfile( + file_id=file_id, + filename=filename, + varname=v, + time_dim=time_dim, + horizontal_dims=horizontal_dims, + xarray_ds=ds, + fill_value_key=fill_value_key, + ) + var_dfs.append(sub_df) + + + return var_dfs + + +def get_ds(ds_grib, pid, lev): + try: + ds = ds_grib.sel(paramId=pid, typeOfLevel=lev).to_xarray() + except: + stepType = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev).metadata("stepType")).tolist() + for steps in stepType: + try: + ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).to_xarray() + except: + num_points = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).metadata("numberOfPoints")).tolist() + for points in num_points: + try: + ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points).to_xarray() + except: + units = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points).metadata("stepUnits")).tolist() + for unit in units: + try: + ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit).to_xarray() + except: + dataType = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit).metadata("dataType")).tolist() + for dtype in dataType: + try: + ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype).to_xarray() + except: + gridType = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype).metadata("gridType")).tolist() + for gtype in gridType: + try: + ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype, gridType=gtype).to_xarray() + except: + print("not working!") + + return ds + def __get_variables(data, time_dim, horizontal_dims): # return a list of variable names from the dataset data that have a time dimension @@ -228,4 +303,5 @@ def parse_csv(file_id, filename, specification): model_output_parser = { # global lookup dict "netcdf": parse_netcdf, "csv": parse_csv, + "grib": parse_grib, } From ae3a747aefaf6ec8b89a0316a6ab18cf65fe1862 Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Fri, 19 Jul 2024 11:06:11 +0200 Subject: [PATCH 02/62] eccodes_definition in setup_env --- setup_env.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/setup_env.sh b/setup_env.sh index f0b3bd69..2d0280cd 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -106,3 +106,14 @@ else fi fi fi + + +# setting ECCODES_DEFINITION_PATH: +git clone --recursive https://github.com/COSMO-ORG/eccodes-cosmo-resources.git + +conda_loc=${CONDA_PREFIX} +base_dir=$(pwd) +def_path_default=${conda_loc}/share/eccodes/definitions +def_path_resources=${base_dir}/eccodes-cosmo-resources/definitions + +conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} \ No newline at end of file From 56ab73e7360f6c49a1b672ef71848de20796b77b Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Mon, 29 Jul 2024 14:12:32 +0200 Subject: [PATCH 03/62] applied changes for reading grib files --- setup_env.sh | 4 +++- util/model_output_parser.py | 5 ++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 2d0280cd..78f18e65 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -111,9 +111,11 @@ fi # setting ECCODES_DEFINITION_PATH: git clone --recursive https://github.com/COSMO-ORG/eccodes-cosmo-resources.git +${CONDA} activate ${DEV_ENV_NAME} conda_loc=${CONDA_PREFIX} base_dir=$(pwd) def_path_default=${conda_loc}/share/eccodes/definitions def_path_resources=${base_dir}/eccodes-cosmo-resources/definitions -conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} \ No newline at end of file +conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} +${CONDA} deactivate \ No newline at end of file diff --git a/util/model_output_parser.py b/util/model_output_parser.py index 5aa253ee..1808a022 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -71,10 +71,9 @@ def parse_grib(file_id, filename, specification): short_name_excl = specification["var_excl"] short_name = np.unique(ds_grib.metadata("shortName")) - short_name = short_name[np.where(~np.isin(short_name, short_name_excl))].tolist() + short_name = short_name[np.isin(short_name, short_name_excl, invert=True, assume_unique=True)].tolist() level_type = np.unique(ds_grib.metadata("typeOfLevel")).tolist() - level_type.remove("unknown") var_dfs = [] @@ -131,7 +130,7 @@ def get_ds(ds_grib, pid, lev): try: ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype, gridType=gtype).to_xarray() except: - print("not working!") + print(f"GRIB file of level {lev} and paramId {pid} cannot be read.") return ds From b3bc25eac4276cd9daf9eb54279058eff7d85daa Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Mon, 29 Jul 2024 17:52:22 +0200 Subject: [PATCH 04/62] Formatting --- requirements/dev-requirements.yml | 1 - setup_env.sh | 2 +- util/model_output_parser.py | 114 ++++++++++++++++++++++++------ 3 files changed, 92 insertions(+), 25 deletions(-) diff --git a/requirements/dev-requirements.yml b/requirements/dev-requirements.yml index e9f7a4a1..22cff43f 100644 --- a/requirements/dev-requirements.yml +++ b/requirements/dev-requirements.yml @@ -21,7 +21,6 @@ dependencies: - sphinx - sphinx-autobuild - toml>=0.10.2 - - pip - earthkit-data - pip: - flake8-pyproject diff --git a/setup_env.sh b/setup_env.sh index 78f18e65..4e1edd84 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -118,4 +118,4 @@ def_path_default=${conda_loc}/share/eccodes/definitions def_path_resources=${base_dir}/eccodes-cosmo-resources/definitions conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} -${CONDA} deactivate \ No newline at end of file +${CONDA} deactivate diff --git a/util/model_output_parser.py b/util/model_output_parser.py index 1808a022..b25cf4ad 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -1,10 +1,10 @@ import sys from collections.abc import Iterable +import earthkit.data import numpy as np import pandas as pd import xarray -import earthkit.data from util.constants import compute_statistics from util.log_handler import logger @@ -61,6 +61,7 @@ def parse_netcdf(file_id, filename, specification): ds.close() return var_dfs + def parse_grib(file_id, filename, specification): logger.debug("parse GRIB file {}".format(filename)) time_dim = specification["time_dim"] @@ -71,19 +72,26 @@ def parse_grib(file_id, filename, specification): short_name_excl = specification["var_excl"] short_name = np.unique(ds_grib.metadata("shortName")) - short_name = short_name[np.isin(short_name, short_name_excl, invert=True, assume_unique=True)].tolist() + short_name = short_name[ + np.isin(short_name, short_name_excl, invert=True, assume_unique=True) + ].tolist() level_type = np.unique(ds_grib.metadata("typeOfLevel")).tolist() - var_dfs = [] for lev in level_type: - paramId = np.unique(ds_grib.sel(typeOfLevel=lev, shortName=short_name).metadata("paramId")).tolist() + paramId = np.unique( + ds_grib.sel(typeOfLevel=lev, shortName=short_name).metadata("paramId") + ).tolist() for pid in paramId: ds_temp = get_ds(ds_grib, pid, lev) v = list(ds_temp.keys())[0] - dim_to_squeeze = [dim for dim, size in zip(ds_temp[v].dims, ds_temp[v].shape) if size==1 and dim != time_dim] + dim_to_squeeze = [ + dim + for dim, size in zip(ds_temp[v].dims, ds_temp[v].shape) + if size == 1 and dim != time_dim + ] ds = ds_temp.squeeze(dim=dim_to_squeeze) sub_df = dataframe_from_ncfile( @@ -97,40 +105,100 @@ def parse_grib(file_id, filename, specification): ) var_dfs.append(sub_df) - return var_dfs def get_ds(ds_grib, pid, lev): try: ds = ds_grib.sel(paramId=pid, typeOfLevel=lev).to_xarray() - except: - stepType = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev).metadata("stepType")).tolist() + except KeyError: + stepType = np.unique( + ds_grib.sel(paramId=pid, typeOfLevel=lev).metadata("stepType") + ).tolist() for steps in stepType: try: - ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).to_xarray() - except: - num_points = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).metadata("numberOfPoints")).tolist() + ds = ds_grib.sel( + paramId=pid, typeOfLevel=lev, stepType=steps + ).to_xarray() + except KeyError: + num_points = np.unique( + ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).metadata( + "numberOfPoints" + ) + ).tolist() for points in num_points: try: - ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points).to_xarray() - except: - units = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points).metadata("stepUnits")).tolist() + ds = ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + ).to_xarray() + except KeyError: + units = np.unique( + ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + ).metadata("stepUnits") + ).tolist() for unit in units: try: - ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit).to_xarray() - except: - dataType = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit).metadata("dataType")).tolist() + ds = ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + stepUnits=unit, + ).to_xarray() + except KeyError: + dataType = np.unique( + ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + stepUnits=unit, + ).metadata("dataType") + ).tolist() for dtype in dataType: try: - ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype).to_xarray() - except: - gridType = np.unique(ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype).metadata("gridType")).tolist() + ds = ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + stepUnits=unit, + dataType=dtype, + ).to_xarray() + except KeyError: + gridType = np.unique( + ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + stepUnits=unit, + dataType=dtype, + ).metadata("gridType") + ).tolist() for gtype in gridType: try: - ds = ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps, numberOfPoints=points, stepUnits=unit, dataType=dtype, gridType=gtype).to_xarray() - except: - print(f"GRIB file of level {lev} and paramId {pid} cannot be read.") + ds = ds_grib.sel( + paramId=pid, + typeOfLevel=lev, + stepType=steps, + numberOfPoints=points, + stepUnits=unit, + dataType=dtype, + gridType=gtype, + ).to_xarray() + except KeyError: + print( + f"GRIB file of level {lev} and" + "paramId {pid} cannot be read." + ) return ds From 325078352ff848e72b5952b3d0b3161d454f09f8 Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Tue, 30 Jul 2024 13:17:16 +0200 Subject: [PATCH 05/62] dummy change --- util/model_output_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/util/model_output_parser.py b/util/model_output_parser.py index b25cf4ad..8985fbe6 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -77,6 +77,7 @@ def parse_grib(file_id, filename, specification): ].tolist() level_type = np.unique(ds_grib.metadata("typeOfLevel")).tolist() + # dummy change var_dfs = [] for lev in level_type: From 940f1f9cfb948ec3aa5a029e984cdbe76a4288b0 Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Tue, 30 Jul 2024 13:20:34 +0200 Subject: [PATCH 06/62] revert dummy change --- util/model_output_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/util/model_output_parser.py b/util/model_output_parser.py index 8985fbe6..b25cf4ad 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -77,7 +77,6 @@ def parse_grib(file_id, filename, specification): ].tolist() level_type = np.unique(ds_grib.metadata("typeOfLevel")).tolist() - # dummy change var_dfs = [] for lev in level_type: From fce5ac039d68c0fb4ce985693d3b684e73c2e64e Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Mon, 5 Aug 2024 14:59:27 +0200 Subject: [PATCH 07/62] adapt get_ds() --- util/model_output_parser.py | 50 +++++++++++++++++++++---------------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/util/model_output_parser.py b/util/model_output_parser.py index b25cf4ad..ec808d0f 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -84,33 +84,36 @@ def parse_grib(file_id, filename, specification): ds_grib.sel(typeOfLevel=lev, shortName=short_name).metadata("paramId") ).tolist() for pid in paramId: - ds_temp = get_ds(ds_grib, pid, lev) - v = list(ds_temp.keys())[0] - - dim_to_squeeze = [ - dim - for dim, size in zip(ds_temp[v].dims, ds_temp[v].shape) - if size == 1 and dim != time_dim - ] - ds = ds_temp.squeeze(dim=dim_to_squeeze) - - sub_df = dataframe_from_ncfile( - file_id=file_id, - filename=filename, - varname=v, - time_dim=time_dim, - horizontal_dims=horizontal_dims, - xarray_ds=ds, - fill_value_key=fill_value_key, - ) - var_dfs.append(sub_df) + ds_temp_list = get_ds(ds_grib, pid, lev) + for ds_temp in ds_temp_list: + v = list(ds_temp.keys())[0] + + dim_to_squeeze = [ + dim + for dim, size in zip(ds_temp[v].dims, ds_temp[v].shape) + if size == 1 and dim != time_dim + ] + ds = ds_temp.squeeze(dim=dim_to_squeeze) + + sub_df = dataframe_from_ncfile( + file_id=file_id, + filename=filename, + varname=v, + time_dim=time_dim, + horizontal_dims=horizontal_dims, + xarray_ds=ds, + fill_value_key=fill_value_key, + ) + var_dfs.append(sub_df) return var_dfs def get_ds(ds_grib, pid, lev): + ds_list = [] try: ds = ds_grib.sel(paramId=pid, typeOfLevel=lev).to_xarray() + ds_list.append(ds) except KeyError: stepType = np.unique( ds_grib.sel(paramId=pid, typeOfLevel=lev).metadata("stepType") @@ -120,6 +123,7 @@ def get_ds(ds_grib, pid, lev): ds = ds_grib.sel( paramId=pid, typeOfLevel=lev, stepType=steps ).to_xarray() + ds_list.append(ds) except KeyError: num_points = np.unique( ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).metadata( @@ -134,6 +138,7 @@ def get_ds(ds_grib, pid, lev): stepType=steps, numberOfPoints=points, ).to_xarray() + ds_list.append(ds) except KeyError: units = np.unique( ds_grib.sel( @@ -152,6 +157,7 @@ def get_ds(ds_grib, pid, lev): numberOfPoints=points, stepUnits=unit, ).to_xarray() + ds_list.append(ds) except KeyError: dataType = np.unique( ds_grib.sel( @@ -172,6 +178,7 @@ def get_ds(ds_grib, pid, lev): stepUnits=unit, dataType=dtype, ).to_xarray() + ds_list.append(ds) except KeyError: gridType = np.unique( ds_grib.sel( @@ -194,13 +201,14 @@ def get_ds(ds_grib, pid, lev): dataType=dtype, gridType=gtype, ).to_xarray() + ds_list.append(ds) except KeyError: print( f"GRIB file of level {lev} and" "paramId {pid} cannot be read." ) - return ds + return ds_list def __get_variables(data, time_dim, horizontal_dims): From b046756c1676cc87af742bb6e0aff91a13b4b516 Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Wed, 7 Aug 2024 15:37:44 +0200 Subject: [PATCH 08/62] unitest and change in grib_definition_path --- setup_env.sh | 7 +-- templates/ICON.jinja | 2 +- tests/engine/test_stats.py | 99 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 104 insertions(+), 4 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 4e1edd84..68540aaa 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -109,13 +109,14 @@ fi # setting ECCODES_DEFINITION_PATH: -git clone --recursive https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${CONDA} activate ${DEV_ENV_NAME} conda_loc=${CONDA_PREFIX} base_dir=$(pwd) def_path_default=${conda_loc}/share/eccodes/definitions -def_path_resources=${base_dir}/eccodes-cosmo-resources/definitions +def_path_icon=/mch-environment/v6/linux-sles15-zen3/nvhpc-24.1/eccodes-2.25.0-qz22mnmqwecsrtnxvldprec5n4xeet4f/share/eccodes/definitions +sample_path=/mch-environment/v6/linux-sles15-zen3/nvhpc-24.1/eccodes-2.25.0-qz22mnmqwecsrtnxvldprec5n4xeet4f/share/eccodes/samples -conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} +conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_icon} +conda env config vars set ECCODES_SAMPLES_PATH=${sample_path} ${CONDA} deactivate diff --git a/templates/ICON.jinja b/templates/ICON.jinja index 2d359ff1..2ff2cec7 100644 --- a/templates/ICON.jinja +++ b/templates/ICON.jinja @@ -11,7 +11,7 @@ "member_type": "{{member_type}}", "factor": 5, "file_specification": [{ - "GRIB": {"format": "GRIB", "time_dim": "step", "horizontal_dims": ["values"], "var_excl": ["tlon", "tlat", "vlon", "vlat", "ulon", "ulat", "h", "slor", "anor", "isor", "sdor"]}, + "GRIB": {"format": "GRIB", "time_dim": "step", "horizontal_dims": ["values"], "var_excl": ["tlon", "tlat", "vlon", "vlat", "ulon", "ulat", "h", "slor", "anor", "isor", "sdor"], "fill_value_key": "_FillValue"}, "latlon": { "format": "netcdf", "time_dim": "time", "horizontal_dims": ["lat:lon"] }, "meteogram": { "format": "netcdf", "time_dim": "time", "horizontal_dims": ["max_nlevs:nstations", "nstations"] }, "dace":{ "format": "netcdf", "time_dim": null, "horizontal_dims": ["d_body"]}, diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 426c5010..42bcf18b 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -1,6 +1,7 @@ import os import unittest +import eccodes import numpy as np from netCDF4 import Dataset @@ -10,6 +11,11 @@ HOR_DIM_SIZE = 100 HEIGHT_DIM_SIZE = 5 +TIME_DIM_GRIB_SIZE = 1 +STEP_DIM_SIZE = 1 +HEIGHT_DIM_GRIB_SIZE = 1 +HORIZONTAL_DIM_GRIB_SIZE = 6114 + def initialize_dummy_netcdf_file(name): data = Dataset(name, "w") @@ -29,6 +35,99 @@ def initialize_dummy_netcdf_file(name): return data +def add_variable_to_grib(filename, dict_data): + with open(filename, "wb") as f_out: + for shortName in list(dict_data.keys()): + gid = eccodes.codes_grib_new_from_samples( + "reduced_rotated_gg_sfc_grib2.tmpl" + ) + eccodes.codes_set(gid, "edition", 2) + eccodes.codes_set(gid, "centre", "lssw") + eccodes.codes_set(gid, "dataDate", 20230913) + eccodes.codes_set(gid, "dataTime", 0) + eccodes.codes_set(gid, "stepRange", 0) + eccodes.codes_set(gid, "typeOfLevel", "surface") + eccodes.codes_set(gid, "level", 0) + eccodes.codes_set(gid, "shortName", shortName) + eccodes.codes_set_values(gid, dict_data[shortName]) + eccodes.codes_write(gid, f_out) + eccodes.codes_release(gid) + + +class TestStatsGrib(unittest.TestCase): + grib_filename = "test_stats_grib.grib" + stats_file_names = "test_stats.csv" + + def setUp(self): + array_t = np.ones( + ( + TIME_DIM_GRIB_SIZE, + STEP_DIM_SIZE, + HEIGHT_DIM_GRIB_SIZE, + HORIZONTAL_DIM_GRIB_SIZE, + ) + ) + array_t[:, :, :, 0] = 0 + array_t[:, :, :, -1] = 2 + + array_pres = ( + np.ones( + ( + TIME_DIM_GRIB_SIZE, + STEP_DIM_SIZE, + HEIGHT_DIM_GRIB_SIZE, + HORIZONTAL_DIM_GRIB_SIZE, + ) + ) + * 3 + ) + array_pres[:, :, :, 0] = 2 + array_pres[:, :, :, -1] = 4 + + dict_data = {"t": array_pres, "v": array_t} + + add_variable_to_grib(self.grib_filename, dict_data) + + def test_stats(self): + file_specification = { + "Test data": dict( + format="grib", + time_dim="step", + horizontal_dims=["values"], + var_excl=[], + fill_value_key="_FillValue", # This should be the name for fill_value. + ), + } + + df = create_stats_dataframe( + input_dir=".", + file_id=[["Test data", self.grib_filename]], + stats_file_name=self.stats_file_names, + file_specification=file_specification, + ) + + # check that the mean/max/min are correct + expected = np.array( + [ + [ + 3.0, + 4.0, + 2.0, + ], + [ + 1.0, + 2.0, + 0.0, + ], + ] + ) + + self.assertTrue( + np.array_equal(df.values, expected), + "stats dataframe incorrect. Difference:\n{}".format(df.values == expected), + ) + + class TestStatsNetcdf(unittest.TestCase): nc_file_name = "test_stats.nc" nc_file_glob = "test_s*.nc" From dde8b24eda6d3e491c223e95023e924caf0a0c40 Mon Sep 17 00:00:00 2001 From: Andrea Leuthard Date: Wed, 7 Aug 2024 16:01:14 +0200 Subject: [PATCH 09/62] old def_path for safety --- setup_env.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 68540aaa..4e1edd84 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -109,14 +109,13 @@ fi # setting ECCODES_DEFINITION_PATH: +git clone --recursive https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${CONDA} activate ${DEV_ENV_NAME} conda_loc=${CONDA_PREFIX} base_dir=$(pwd) def_path_default=${conda_loc}/share/eccodes/definitions -def_path_icon=/mch-environment/v6/linux-sles15-zen3/nvhpc-24.1/eccodes-2.25.0-qz22mnmqwecsrtnxvldprec5n4xeet4f/share/eccodes/definitions -sample_path=/mch-environment/v6/linux-sles15-zen3/nvhpc-24.1/eccodes-2.25.0-qz22mnmqwecsrtnxvldprec5n4xeet4f/share/eccodes/samples +def_path_resources=${base_dir}/eccodes-cosmo-resources/definitions -conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_icon} -conda env config vars set ECCODES_SAMPLES_PATH=${sample_path} +conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} ${CONDA} deactivate From 654622345c09962ebb5499740cb4a9f755ca2f68 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 1 Oct 2024 10:06:16 +0200 Subject: [PATCH 10/62] improve formatting --- .pre-commit-config.yaml | 2 +- tests/engine/test_stats.py | 23 ++++++++++++----------- util/model_output_parser.py | 18 +++++++++--------- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ae63ad84..5d12020f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -121,7 +121,7 @@ repos: types: [python] args: - "--max-line-length=88" - - "--disable=C0116,R0912,R0913,R0914,R0915,R1710,W0511,W0719" + - "--disable=C0116,R0912,R0913,R0914,R0915,R0917,R1710,W0511,W0719" - repo: local hooks: - id: flake8 diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 382cbe25..69335c20 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -43,7 +43,7 @@ def initialize_dummy_netcdf_file(name): def add_variable_to_grib(filename, dict_data): with open(filename, "wb") as f_out: - for shortName in list(dict_data.keys()): + for short_name in list(dict_data.keys()): gid = eccodes.codes_grib_new_from_samples( "reduced_rotated_gg_sfc_grib2.tmpl" ) @@ -54,8 +54,8 @@ def add_variable_to_grib(filename, dict_data): eccodes.codes_set(gid, "stepRange", 0) eccodes.codes_set(gid, "typeOfLevel", "surface") eccodes.codes_set(gid, "level", 0) - eccodes.codes_set(gid, "shortName", shortName) - eccodes.codes_set_values(gid, dict_data[shortName]) + eccodes.codes_set(gid, "shortName", short_name) + eccodes.codes_set_values(gid, dict_data[short_name]) eccodes.codes_write(gid, f_out) eccodes.codes_release(gid) @@ -96,13 +96,14 @@ def setUp(self): def test_stats(self): file_specification = { - "Test data": dict( - format="grib", - time_dim="step", - horizontal_dims=["values"], - var_excl=[], - fill_value_key="_FillValue", # This should be the name for fill_value. - ), + "Test data": { + "format": "grib", + "time_dim": "step", + "horizontal_dims": ["values"], + "var_excl": [], + "fill_value_key": "_FillValue", + # This should be the name for fill_value. + }, } df = create_stats_dataframe( @@ -130,7 +131,7 @@ def test_stats(self): self.assertTrue( np.array_equal(df.values, expected), - "stats dataframe incorrect. Difference:\n{}".format(df.values == expected), + f"stats dataframe incorrect. Difference:\n{df.values == expected}", ) diff --git a/util/model_output_parser.py b/util/model_output_parser.py index a2005a95..b1cc400f 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -65,7 +65,7 @@ def parse_netcdf(file_id, filename, specification): def parse_grib(file_id, filename, specification): - logger.debug("parse GRIB file {}".format(filename)) + logger.debug("parse GRIB file %s", filename) time_dim = specification["time_dim"] horizontal_dims = specification["horizontal_dims"] fill_value_key = specification.get("fill_value_key", None) @@ -82,10 +82,10 @@ def parse_grib(file_id, filename, specification): var_dfs = [] for lev in level_type: - paramId = np.unique( + param_id = np.unique( ds_grib.sel(typeOfLevel=lev, shortName=short_name).metadata("paramId") ).tolist() - for pid in paramId: + for pid in param_id: ds_temp_list = get_ds(ds_grib, pid, lev) for ds_temp in ds_temp_list: v = list(ds_temp.keys())[0] @@ -117,10 +117,10 @@ def get_ds(ds_grib, pid, lev): ds = ds_grib.sel(paramId=pid, typeOfLevel=lev).to_xarray() ds_list.append(ds) except KeyError: - stepType = np.unique( + step_type = np.unique( ds_grib.sel(paramId=pid, typeOfLevel=lev).metadata("stepType") ).tolist() - for steps in stepType: + for steps in step_type: try: ds = ds_grib.sel( paramId=pid, typeOfLevel=lev, stepType=steps @@ -161,7 +161,7 @@ def get_ds(ds_grib, pid, lev): ).to_xarray() ds_list.append(ds) except KeyError: - dataType = np.unique( + data_type = np.unique( ds_grib.sel( paramId=pid, typeOfLevel=lev, @@ -170,7 +170,7 @@ def get_ds(ds_grib, pid, lev): stepUnits=unit, ).metadata("dataType") ).tolist() - for dtype in dataType: + for dtype in data_type: try: ds = ds_grib.sel( paramId=pid, @@ -182,7 +182,7 @@ def get_ds(ds_grib, pid, lev): ).to_xarray() ds_list.append(ds) except KeyError: - gridType = np.unique( + grid_type = np.unique( ds_grib.sel( paramId=pid, typeOfLevel=lev, @@ -192,7 +192,7 @@ def get_ds(ds_grib, pid, lev): dataType=dtype, ).metadata("gridType") ).tolist() - for gtype in gridType: + for gtype in grid_type: try: ds = ds_grib.sel( paramId=pid, From d3a942ee35190ef1c9f654c019776faed965d248 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 1 Oct 2024 12:29:58 +0200 Subject: [PATCH 11/62] frist draft tests --- tests/engine/test_stats.py | 124 +++++++++++++++++------------------- util/model_output_parser.py | 41 ++++++++++-- 2 files changed, 96 insertions(+), 69 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 69335c20..ec3259d5 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -9,6 +9,7 @@ import eccodes import numpy as np +import pytest from netCDF4 import Dataset # pylint: disable=no-name-in-module from engine.stats import create_stats_dataframe @@ -22,6 +23,9 @@ HEIGHT_DIM_GRIB_SIZE = 1 HORIZONTAL_DIM_GRIB_SIZE = 6114 +GRIB_FILENAME = "test_stats_grib.grib" +STATS_FILE_NAMES = "test_stats.csv" + def initialize_dummy_netcdf_file(name): data = Dataset(name, "w") @@ -60,12 +64,21 @@ def add_variable_to_grib(filename, dict_data): eccodes.codes_release(gid) -class TestStatsGrib(unittest.TestCase): - grib_filename = "test_stats_grib.grib" - stats_file_names = "test_stats.csv" +@pytest.fixture +def setup_grib_file(): + array_t = np.ones( + ( + TIME_DIM_GRIB_SIZE, + STEP_DIM_SIZE, + HEIGHT_DIM_GRIB_SIZE, + HORIZONTAL_DIM_GRIB_SIZE, + ) + ) + array_t[:, :, :, 0] = 0 + array_t[:, :, :, -1] = 2 - def setUp(self): - array_t = np.ones( + array_pres = ( + np.ones( ( TIME_DIM_GRIB_SIZE, STEP_DIM_SIZE, @@ -73,66 +86,47 @@ def setUp(self): HORIZONTAL_DIM_GRIB_SIZE, ) ) - array_t[:, :, :, 0] = 0 - array_t[:, :, :, -1] = 2 - - array_pres = ( - np.ones( - ( - TIME_DIM_GRIB_SIZE, - STEP_DIM_SIZE, - HEIGHT_DIM_GRIB_SIZE, - HORIZONTAL_DIM_GRIB_SIZE, - ) - ) - * 3 - ) - array_pres[:, :, :, 0] = 2 - array_pres[:, :, :, -1] = 4 - - dict_data = {"t": array_pres, "v": array_t} - - add_variable_to_grib(self.grib_filename, dict_data) - - def test_stats(self): - file_specification = { - "Test data": { - "format": "grib", - "time_dim": "step", - "horizontal_dims": ["values"], - "var_excl": [], - "fill_value_key": "_FillValue", - # This should be the name for fill_value. - }, - } - - df = create_stats_dataframe( - input_dir=".", - file_id=[["Test data", self.grib_filename]], - stats_file_name=self.stats_file_names, - file_specification=file_specification, - ) - - # check that the mean/max/min are correct - expected = np.array( - [ - [ - 3.0, - 4.0, - 2.0, - ], - [ - 1.0, - 2.0, - 0.0, - ], - ] - ) - - self.assertTrue( - np.array_equal(df.values, expected), - f"stats dataframe incorrect. Difference:\n{df.values == expected}", - ) + * 3 + ) + array_pres[:, :, :, 0] = 2 + array_pres[:, :, :, -1] = 4 + + dict_data = {"t": array_pres, "v": array_t} + + # This would be where your grib file is created + add_variable_to_grib(GRIB_FILENAME, dict_data) + + +@pytest.mark.usefixtures("setup_grib_file") +def test_stats(): + file_specification = { + "Test data": { + "format": "grib", + "time_dim": "step", + "horizontal_dims": ["values"], + "var_excl": [], + "fill_value_key": "_FillValue", # This should be the name for fill_value. + }, + } + + df = create_stats_dataframe( + input_dir=".", + file_id=[["Test data", GRIB_FILENAME]], + stats_file_name=STATS_FILE_NAMES, + file_specification=file_specification, + ) + + # check that the mean/max/min are correct + expected = np.array( + [ + [3.0, 4.0, 2.0], + [1.0, 2.0, 0.0], + ] + ) + + assert np.array_equal( + df.values, expected + ), f"Stats dataframe incorrect. Difference:\n{df.values == expected}" class TestStatsNetcdf(unittest.TestCase): diff --git a/util/model_output_parser.py b/util/model_output_parser.py index b1cc400f..f73bb711 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -112,6 +112,38 @@ def parse_grib(file_id, filename, specification): def get_ds(ds_grib, pid, lev): + """ + Retrieve datasets from a GRIB file based on specified parameters and + hierarchical metadata. + + This function attempts to extract data from the GRIB file by selecting + fields that match the given `paramId` and `typeOfLevel`. + If the initial selection fails due to missing or mismatched metadata, the + function will recursively explore other metadata fields such as `stepType`, + `numberOfPoints`, `stepUnits`, `dataType`, and `gridType` to find matching + datasets. + + Parameters: + ----------- + ds_grib : GRIB object + The GRIB file object to extract data from. + pid : int + The parameter ID (`paramId`) to select in the GRIB file. + lev : str + The level type (`typeOfLevel`) to select in the GRIB file. + + Returns: + -------- + ds_list : list + A list of xarray datasets that match the specified parameter and level, + with additional filtering based on hierarchical metadata fields. + + Notes: + ------ + - The function handles `KeyError` exceptions by recursively selecting data + with additional metadata fields (e.g., stepType, numberOfPoints, etc.). + - If no matching datasets are found, the function prints an error message. + """ ds_list = [] try: ds = ds_grib.sel(paramId=pid, typeOfLevel=lev).to_xarray() @@ -205,11 +237,12 @@ def get_ds(ds_grib, pid, lev): ).to_xarray() ds_list.append(ds) except KeyError: - print( - f"GRIB file of level {lev} and" - "paramId {pid} cannot be read." + logger.warning( + "GRIB file of level %s and " + "paramId %s cannot be read.", + lev, + pid, ) - return ds_list From f62a22a7fdfb202d4ecdb93689feda7d172fbf3b Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 1 Oct 2024 14:38:30 +0200 Subject: [PATCH 12/62] add unittest --- tests/util/test_model_output_parser.py | 96 ++++++++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 tests/util/test_model_output_parser.py diff --git a/tests/util/test_model_output_parser.py b/tests/util/test_model_output_parser.py new file mode 100644 index 00000000..d33d185c --- /dev/null +++ b/tests/util/test_model_output_parser.py @@ -0,0 +1,96 @@ +""" +This module contains unit tests for the `model_output_parser` module. +""" + +from unittest.mock import MagicMock + +import pytest + +from util.model_output_parser import get_ds + + +@pytest.fixture(name="mock_ds_grib") +def fixture_mock_ds_grib(): + """ + Fixture that creates a mock GRIB object to simulate different cases of data + selection and exceptions that `get_ds` needs to handle. + """ + ds_grib = MagicMock() + + # Simulating the successful selection and conversion to xarray + ds_grib.sel.return_value.to_xarray.return_value = "valid_xarray" + + # Simulating the metadata retrieval (unique values) + ds_grib.sel.return_value.metadata.side_effect = [ + ["forecast"], # stepType + [100], # numberOfPoints + ["hours"], # stepUnits + ["analysis"], # dataType + ["regular_ll"], # gridType + ] + + return ds_grib + + +def test_get_ds_success(mock_ds_grib): + """ + Test case where get_ds successfully retrieves the dataset on the first attempt. + """ + pid = 1 + lev = "surface" + + result = get_ds(mock_ds_grib, pid, lev) + + # Ensure the dataset is selected once + mock_ds_grib.sel.assert_called_once_with(paramId=pid, typeOfLevel=lev) + + # The result should contain the mocked xarray dataset + assert result == ["valid_xarray"] + + +@pytest.mark.parametrize( + "to_xarray_return_value", + [ + (KeyError(), "valid_xarray"), + (KeyError(), KeyError(), "valid_xarray"), + (KeyError(), KeyError(), KeyError(), "valid_xarray"), + (KeyError(), KeyError(), KeyError(), KeyError(), "valid_xarray"), + (KeyError(), KeyError(), KeyError(), KeyError(), KeyError(), "valid_xarray"), + ], +) +def test_get_ds_recursive_selection(mock_ds_grib, to_xarray_return_value): + """ + Test case where get_ds recursively selects the dataset by metadata fields. + """ + pid = 1 + lev = "surface" + + mock_ds_grib.sel.return_value.to_xarray.side_effect = to_xarray_return_value + + result = get_ds(mock_ds_grib, pid, lev) + + # Ensure the recursive logic is triggered by calling sel multiple times + assert mock_ds_grib.sel.call_count >= len(to_xarray_return_value) + + # The result should contain the mocked xarray dataset + assert result == ["valid_xarray"] + + +def test_get_ds_keyerror_handling(caplog, mock_ds_grib): + """ + Test case where get_ds fails to retrieve data and handles multiple KeyErrors. + """ + # # Create a mock GRIB object + # mock_ds_grib = MagicMock() + + pid = 1 + lev = "surface" + + # Simulate KeyErrors for all attempts to select datasets + mock_ds_grib.sel.return_value.to_xarray.side_effect = KeyError() + + result = get_ds(mock_ds_grib, pid, lev) + + # Assert that the warning was logged + assert "GRIB file of level surface and paramId 1 cannot be read." in caplog.text + assert not result From 35e982140207140bde3a34912783f2f18fde9016 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 1 Oct 2024 14:43:03 +0200 Subject: [PATCH 13/62] rm deleted files --- requirements/dev-environment.yml | 260 ------------------------------ requirements/dev-requirements.yml | 26 --- 2 files changed, 286 deletions(-) delete mode 100644 requirements/dev-environment.yml delete mode 100644 requirements/dev-requirements.yml diff --git a/requirements/dev-environment.yml b/requirements/dev-environment.yml deleted file mode 100644 index 6ac4dddb..00000000 --- a/requirements/dev-environment.yml +++ /dev/null @@ -1,260 +0,0 @@ -name: probtest-dev -channels: - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1 - - _openmp_mutex=4.5 - - alabaster=0.7.12 - - alsa-lib=1.2.8 - - appdirs=1.4.4 - - astroid=2.13.2 - - asttokens=2.2.1 - - attr=2.5.1 - - attrs=22.2.0 - - babel=2.11.0 - - backcall=0.2.0 - - backports=1.0 - - backports.functools_lru_cache=1.6.4 - - black=22.10.0 - - brotli=1.0.9 - - brotli-bin=1.0.9 - - brotlipy=0.7.0 - - bzip2=1.0.8 - - c-ares=1.18.1 - - ca-certificates=2022.12.7 - - certifi=2022.12.7 - - cffi=1.15.1 - - cfgv=3.3.1 - - cftime=1.6.2 - - charset-normalizer=2.1.1 - - click=8.1.3 - - codespell=2.2.2 - - colorama=0.4.6 - - contourpy=1.0.6 - - cryptography=39.0.0 - - curl=7.87.0 - - cycler=0.11.0 - - dbus=1.13.6 - - decorator=5.1.1 - - dill=0.3.6 - - distlib=0.3.6 - - docutils=0.17.1 - - exceptiongroup=1.1.0 - - executing=1.2.0 - - expat=2.5.0 - - fftw=3.3.10 - - filelock=3.9.0 - - flake8=6.0.0 - - flake8-black=0.3.5 - - font-ttf-dejavu-sans-mono=2.37 - - font-ttf-inconsolata=3.000 - - font-ttf-source-code-pro=2.038 - - font-ttf-ubuntu=0.83 - - fontconfig=2.14.1 - - fonts-conda-ecosystem=1 - - fonts-conda-forge=1 - - fonttools=4.38.0 - - freetype=2.12.1 - - gettext=0.21.1 - - glib=2.74.1 - - glib-tools=2.74.1 - - gst-plugins-base=1.21.3 - - gstreamer=1.21.3 - - gstreamer-orc=0.4.33 - - hdf4=4.2.15 - - hdf5=1.12.2 - - icu=70.1 - - identify=2.5.12 - - idna=3.4 - - imagesize=1.4.1 - - importlib-metadata=3.3.0 - - iniconfig=2.0.0 - - ipdb=0.13.11 - - ipython=8.8.0 - - isort=5.11.4 - - jack=1.9.21 - - jedi=0.18.2 - - jinja2=3.1.2 - - jpeg=9e - - keyutils=1.6.1 - - kiwisolver=1.4.4 - - krb5=1.20.1 - - lame=3.100 - - lazy-object-proxy=1.9.0 - - lcms2=2.14 - - ld_impl_linux-64=2.39 - - lerc=4.0.0 - - libaec=1.0.6 - - libblas=3.9.0 - - libbrotlicommon=1.0.9 - - libbrotlidec=1.0.9 - - libbrotlienc=1.0.9 - - libcap=2.66 - - libcblas=3.9.0 - - libclang=15.0.6 - - libclang13=15.0.6 - - libcups=2.3.3 - - libcurl=7.87.0 - - libdb=6.2.32 - - libdeflate=1.14 - - libedit=3.1.20191231 - - libev=4.33 - - libevent=2.1.10 - - libffi=3.4.2 - - libflac=1.4.2 - - libgcc-ng=12.2.0 - - libgcrypt=1.10.1 - - libgfortran-ng=12.2.0 - - libgfortran5=12.2.0 - - libglib=2.74.1 - - libgomp=12.2.0 - - libgpg-error=1.46 - - libiconv=1.17 - - libjpeg-turbo=2.1.4 - - liblapack=3.9.0 - - libllvm15=15.0.6 - - libnetcdf=4.8.1 - - libnghttp2=1.51.0 - - libnsl=2.0.0 - - libogg=1.3.4 - - libopenblas=0.3.21 - - libopus=1.3.1 - - libpng=1.6.39 - - libpq=15.1 - - libsndfile=1.1.0 - - libsqlite=3.40.0 - - libssh2=1.10.0 - - libstdcxx-ng=12.2.0 - - libsystemd0=252 - - libtiff=4.5.0 - - libtool=2.4.7 - - libudev1=252 - - libuuid=2.32.1 - - libvorbis=1.3.7 - - libwebp-base=1.2.4 - - libxcb=1.13 - - libxkbcommon=1.0.3 - - libxml2=2.10.3 - - libzip=1.9.2 - - libzlib=1.2.13 - - livereload=2.6.3 - - lz4-c=1.9.3 - - markupsafe=2.1.1 - - matplotlib=3.6.2 - - matplotlib-base=3.6.2 - - matplotlib-inline=0.1.6 - - mccabe=0.7.0 - - mpg123=1.31.1 - - munkres=1.1.4 - - mypy=0.991 - - mypy_extensions=0.4.3 - - mysql-common=8.0.31 - - mysql-libs=8.0.31 - - ncurses=6.3 - - netcdf4=1.6.2 - - nodeenv=1.7.0 - - nspr=4.35 - - nss=3.82 - - numpy=1.24.1 - - openjpeg=2.5.0 - - openssl=3.0.7 - - packaging=22.0 - - pandas=1.5.2 - - parso=0.8.3 - - pathlib=1.0.1 - - pathspec=0.10.3 - - pcre2=10.40 - - pexpect=4.8.0 - - pickleshare=0.7.5 - - pillow=9.4.0 - - pip=22.3.1 - - platformdirs=2.6.2 - - pluggy=1.0.0 - - ply=3.11 - - pooch=1.6.0 - - pre-commit=2.21.0 - - pre-commit-hooks=4.4.0 - - prompt-toolkit=3.0.36 - - psutil=5.9.4 - - pthread-stubs=0.4 - - ptyprocess=0.7.0 - - pulseaudio=16.1 - - pure_eval=0.2.2 - - pycodestyle=2.10.0 - - pycparser=2.21 - - pydantic=1.10.4 - - pydocstyle=6.2.2 - - pyflakes=3.0.1 - - pygments=2.14.0 - - pylint=2.15.10 - - pyopenssl=23.0.0 - - pyparsing=3.0.9 - - pyqt=5.15.7 - - pyqt5-sip=12.11.0 - - pysocks=1.7.1 - - pytest=7.2.0 - - python=3.11.0 - - python-dateutil=2.8.2 - - python_abi=3.11 - - pytoolconfig=1.2.4 - - pytz=2022.7 - - pyyaml=6.0 - - qt-main=5.15.6 - - readline=8.1.2 - - regex=2022.10.31 - - requests=2.28.1 - - rope=1.6.0 - - rstcheck=6.1.1 - - rstcheck-core=1.0.2 - - ruamel.yaml=0.17.21 - - ruamel.yaml.clib=0.2.7 - - scipy=1.10.0 - - setuptools=65.6.3 - - shellingham=1.5.0.post1 - - sip=6.7.5 - - six=1.16.0 - - snowballstemmer=2.2.0 - - sphinx=4.3.2 - - sphinx-autobuild=2021.3.14 - - sphinxcontrib-applehelp=1.0.2 - - sphinxcontrib-devhelp=1.0.2 - - sphinxcontrib-htmlhelp=2.0.0 - - sphinxcontrib-jsmath=1.0.1 - - sphinxcontrib-qthelp=1.0.3 - - sphinxcontrib-serializinghtml=1.1.5 - - stack_data=0.6.2 - - tk=8.6.12 - - toml=0.10.2 - - tomli=2.0.1 - - tomlkit=0.11.6 - - tornado=6.2 - - traitlets=5.8.1 - - typer=0.4.2 - - types-docutils=0.18.3 - - typing=3.10.0.0 - - typing-extensions=4.4.0 - - typing_extensions=4.4.0 - - tzdata=2022g - - ukkonen=1.0.1 - - urllib3=1.26.13 - - virtualenv=20.17.1 - - wcwidth=0.2.5 - - wheel=0.38.4 - - wrapt=1.14.1 - - xarray=2022.12.0 - - xcb-util=0.4.0 - - xcb-util-image=0.4.0 - - xcb-util-keysyms=0.4.0 - - xcb-util-renderutil=0.3.9 - - xcb-util-wm=0.4.1 - - xorg-libxau=1.0.9 - - xorg-libxdmcp=1.1.3 - - xz=5.2.6 - - yaml=0.2.5 - - zipp=3.11.0 - - zlib=1.2.13 - - zstd=1.5.2 - - earthkit-data - - pip: - - flake8-pyproject==1.2.2 diff --git a/requirements/dev-requirements.yml b/requirements/dev-requirements.yml deleted file mode 100644 index 22cff43f..00000000 --- a/requirements/dev-requirements.yml +++ /dev/null @@ -1,26 +0,0 @@ -channels: - - conda-forge - - defaults -dependencies: - - black>=22.8.0 - - codespell>=2.2.2 - - flake8>=5.0.4 - - flake8-black - - ipdb - - ipython - - isort>=5 - - mypy - - pip - - pre-commit>=2.19.0 - - pre-commit-hooks>=4.3.0 - - pydocstyle>=6.1.1 - - pylint>=2.6.0 - - pytest - - rope - - rstcheck>=6.1.1 - - sphinx - - sphinx-autobuild - - toml>=0.10.2 - - earthkit-data - - pip: - - flake8-pyproject From c72eb96fd2b56af12d3fba74a09f347bf4b3a082 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 1 Oct 2024 17:00:35 +0200 Subject: [PATCH 14/62] update requirements --- requirements/environment.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 642e25d8..6516b500 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -24,10 +24,10 @@ dependencies: - aws-crt-cpp=0.28.3 - aws-sdk-cpp=1.11.407 - azure-core-cpp=1.13.0 - - azure-identity-cpp=1.8.0 - - azure-storage-blobs-cpp=12.12.0 - - azure-storage-common-cpp=12.7.0 - - azure-storage-files-datalake-cpp=12.11.0 + - azure-identity-cpp=1.9.0 + - azure-storage-blobs-cpp=12.13.0 + - azure-storage-common-cpp=12.8.0 + - azure-storage-files-datalake-cpp=12.12.0 - black=24.8.0 - blosc=1.21.6 - bokeh=3.5.2 From 5048f491fcf9616fb90f4ed5e7f8dd9606319d71 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 1 Oct 2024 17:00:35 +0200 Subject: [PATCH 15/62] update requirements --- requirements/environment.yml | 8 +- tests/util/test_model_output_parser.py | 40 ++++++-- util/model_output_parser.py | 134 +++++-------------------- 3 files changed, 60 insertions(+), 122 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 642e25d8..6516b500 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -24,10 +24,10 @@ dependencies: - aws-crt-cpp=0.28.3 - aws-sdk-cpp=1.11.407 - azure-core-cpp=1.13.0 - - azure-identity-cpp=1.8.0 - - azure-storage-blobs-cpp=12.12.0 - - azure-storage-common-cpp=12.7.0 - - azure-storage-files-datalake-cpp=12.11.0 + - azure-identity-cpp=1.9.0 + - azure-storage-blobs-cpp=12.13.0 + - azure-storage-common-cpp=12.8.0 + - azure-storage-files-datalake-cpp=12.12.0 - black=24.8.0 - blosc=1.21.6 - bokeh=3.5.2 diff --git a/tests/util/test_model_output_parser.py b/tests/util/test_model_output_parser.py index d33d185c..e54fee20 100644 --- a/tests/util/test_model_output_parser.py +++ b/tests/util/test_model_output_parser.py @@ -49,16 +49,37 @@ def test_get_ds_success(mock_ds_grib): @pytest.mark.parametrize( - "to_xarray_return_value", + "to_xarray_return_value, expected_result", [ - (KeyError(), "valid_xarray"), - (KeyError(), KeyError(), "valid_xarray"), - (KeyError(), KeyError(), KeyError(), "valid_xarray"), - (KeyError(), KeyError(), KeyError(), KeyError(), "valid_xarray"), - (KeyError(), KeyError(), KeyError(), KeyError(), KeyError(), "valid_xarray"), + ((KeyError(), "valid_stepType_xarray"), ["valid_stepType_xarray"]), + ( + (KeyError(), KeyError(), "valid_numberOfPoints_xarray"), + ["valid_numberOfPoints_xarray"], + ), + ( + (KeyError(), KeyError(), KeyError(), "valid_stepUnits_xarray"), + ["valid_stepUnits_xarray"], + ), + ( + (KeyError(), KeyError(), KeyError(), KeyError(), "valid_dataType_xarray"), + ["valid_dataType_xarray"], + ), + ( + ( + KeyError(), + KeyError(), + KeyError(), + KeyError(), + KeyError(), + "valid_gridType_xarray", + ), + ["valid_gridType_xarray"], + ), ], ) -def test_get_ds_recursive_selection(mock_ds_grib, to_xarray_return_value): +def test_get_ds_recursive_selection( + mock_ds_grib, to_xarray_return_value, expected_result +): """ Test case where get_ds recursively selects the dataset by metadata fields. """ @@ -73,16 +94,13 @@ def test_get_ds_recursive_selection(mock_ds_grib, to_xarray_return_value): assert mock_ds_grib.sel.call_count >= len(to_xarray_return_value) # The result should contain the mocked xarray dataset - assert result == ["valid_xarray"] + assert result == expected_result def test_get_ds_keyerror_handling(caplog, mock_ds_grib): """ Test case where get_ds fails to retrieve data and handles multiple KeyErrors. """ - # # Create a mock GRIB object - # mock_ds_grib = MagicMock() - pid = 1 lev = "surface" diff --git a/util/model_output_parser.py b/util/model_output_parser.py index f73bb711..6db0f315 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -117,11 +117,10 @@ def get_ds(ds_grib, pid, lev): hierarchical metadata. This function attempts to extract data from the GRIB file by selecting - fields that match the given `paramId` and `typeOfLevel`. - If the initial selection fails due to missing or mismatched metadata, the - function will recursively explore other metadata fields such as `stepType`, - `numberOfPoints`, `stepUnits`, `dataType`, and `gridType` to find matching - datasets. + fields that match the given `paramId` and `typeOfLevel`. If the initial + selection fails due to missing or mismatched metadata, the function + will explore other metadata fields such as `stepType`, `numberOfPoints`, + `stepUnits`, `dataType`, and `gridType` to find matching datasets. Parameters: ----------- @@ -137,112 +136,33 @@ def get_ds(ds_grib, pid, lev): ds_list : list A list of xarray datasets that match the specified parameter and level, with additional filtering based on hierarchical metadata fields. - - Notes: - ------ - - The function handles `KeyError` exceptions by recursively selecting data - with additional metadata fields (e.g., stepType, numberOfPoints, etc.). - - If no matching datasets are found, the function prints an error message. """ ds_list = [] - try: - ds = ds_grib.sel(paramId=pid, typeOfLevel=lev).to_xarray() - ds_list.append(ds) - except KeyError: - step_type = np.unique( - ds_grib.sel(paramId=pid, typeOfLevel=lev).metadata("stepType") - ).tolist() - for steps in step_type: + selectors = {"paramId": pid, "typeOfLevel": lev} + metadata_keys = ["stepType", "numberOfPoints", "stepUnits", "dataType", "gridType"] + + def recursive_select(selects, depth=0): + try: + ds = ds_grib.sel(**selects).to_xarray() + ds_list.append(ds) + except KeyError: + if depth == len(metadata_keys): # No more metadata keys to try + return + key = metadata_keys[depth] try: - ds = ds_grib.sel( - paramId=pid, typeOfLevel=lev, stepType=steps - ).to_xarray() - ds_list.append(ds) + values = np.unique(ds_grib.sel(**selects).metadata(key)).tolist() + for value in values: + selects[key] = value + recursive_select(selects, depth + 1) # Recurse to next level except KeyError: - num_points = np.unique( - ds_grib.sel(paramId=pid, typeOfLevel=lev, stepType=steps).metadata( - "numberOfPoints" - ) - ).tolist() - for points in num_points: - try: - ds = ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - ).to_xarray() - ds_list.append(ds) - except KeyError: - units = np.unique( - ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - ).metadata("stepUnits") - ).tolist() - for unit in units: - try: - ds = ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - stepUnits=unit, - ).to_xarray() - ds_list.append(ds) - except KeyError: - data_type = np.unique( - ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - stepUnits=unit, - ).metadata("dataType") - ).tolist() - for dtype in data_type: - try: - ds = ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - stepUnits=unit, - dataType=dtype, - ).to_xarray() - ds_list.append(ds) - except KeyError: - grid_type = np.unique( - ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - stepUnits=unit, - dataType=dtype, - ).metadata("gridType") - ).tolist() - for gtype in grid_type: - try: - ds = ds_grib.sel( - paramId=pid, - typeOfLevel=lev, - stepType=steps, - numberOfPoints=points, - stepUnits=unit, - dataType=dtype, - gridType=gtype, - ).to_xarray() - ds_list.append(ds) - except KeyError: - logger.warning( - "GRIB file of level %s and " - "paramId %s cannot be read.", - lev, - pid, - ) + pass + + # Try initial selection + recursive_select(selectors) + + if not ds_list: + logger.warning("GRIB file of level %s and paramId %s cannot be read.", lev, pid) + return ds_list From f9d04a90d895ad281060b7f974ff6a8381b8c367 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 2 Oct 2024 10:58:21 +0200 Subject: [PATCH 16/62] use pinned version for ci and make python version consistent --- .github/workflows/pre-commit.yml | 6 +++--- .github/workflows/pytest.yml | 2 +- setup_env.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 1c968868..d95d709c 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -19,16 +19,16 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.11.0 + python-version: 3.12.6 - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" channels: conda-forge channel-priority: flexible show-channel-urls: true - - name: Create env from unpinned reqs + - name: Create env from pinned reqs run: | - conda env create --name dev_env --file requirements/requirements.yml + conda env create --name dev_env --file requirements/environment.yml - name: Install pre-commit hooks run: | conda run --name dev_env pre-commit install-hooks diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index d3d2bd58..c9e62b27 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.11.0 + python-version: 3.12.6 - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" diff --git a/setup_env.sh b/setup_env.sh index 9e902679..ccaeab0a 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -13,7 +13,7 @@ DEFAULT_ENV_NAME="probtest" # Default options ENV_NAME="${DEFAULT_ENV_NAME}" -PYVERSION=3.10 +PYVERSION=3.12.6 PINNED=true EXPORT=false CONDA=conda From 145dd49f0870ec76260cdcc38496bec1877eab67 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 2 Oct 2024 12:33:00 +0200 Subject: [PATCH 17/62] pin env and set python version --- .github/workflows/pytest.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index c9e62b27..b91e66bd 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -26,9 +26,9 @@ jobs: channels: conda-forge channel-priority: flexible show-channel-urls: true - - name: Create dev env from unpinned reqs + - name: Create dev env from pinned reqs run: | - conda env create --name dev_env --file requirements/requirements.yml + conda env create --name dev_env --file requirements/environment.yml - name: Run Pytest env: TZ: Europe/Zurich From 7dd8c184b2918c43df9897010c7579dcc2c8816e Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 2 Oct 2024 12:49:34 +0200 Subject: [PATCH 18/62] add author --- AUTHORS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS.md b/AUTHORS.md index 37d6c2f5..f28d533c 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -6,6 +6,7 @@ | Marek Jacob | DWD | | Jonas Jucker | C2SM | | Annika Lauber | C2SM | +| Andrea Leuthard | MeteoSwiss | | Giacomo Serafini | MeteoSwiss | | Mikael Stellio | C2SM | | Ben Weber | MeteoSwiss | From e450ff7d7b33cf370dd2b9cd6eb48ad3274b58ed Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 2 Oct 2024 13:00:19 +0200 Subject: [PATCH 19/62] fix gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 46ee5c08..beb48e22 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ .idea* -.vscode/* -!.vscode/settings.json +.vscode *pycache* *.swp *.pdf From cb5d76cfa57290943fa3e9cdb378c72326d9fa09 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 2 Oct 2024 13:05:44 +0200 Subject: [PATCH 20/62] update gitignore --- .gitignore | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index beb48e22..540f87eb 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +.coverage .idea* .vscode *pycache* @@ -6,8 +7,9 @@ *.png *.log *.nc -.coverage coverage.xml +eccodes-cosmo-resources +miniconda # output from unittest test_stats.csv From 13ce70cedb93dd1cd0c0701801986d4f3bf3725b Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 3 Oct 2024 11:42:48 +0200 Subject: [PATCH 21/62] small improvement --- tests/engine/test_stats.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index ec3259d5..a9549d24 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -65,7 +65,7 @@ def add_variable_to_grib(filename, dict_data): @pytest.fixture -def setup_grib_file(): +def setup_grib_file(tmp_dir): array_t = np.ones( ( TIME_DIM_GRIB_SIZE, @@ -94,11 +94,11 @@ def setup_grib_file(): dict_data = {"t": array_pres, "v": array_t} # This would be where your grib file is created - add_variable_to_grib(GRIB_FILENAME, dict_data) + add_variable_to_grib(os.path.join(tmp_dir, GRIB_FILENAME), dict_data) @pytest.mark.usefixtures("setup_grib_file") -def test_stats(): +def test_stats_grib(tmp_dir): file_specification = { "Test data": { "format": "grib", @@ -110,7 +110,7 @@ def test_stats(): } df = create_stats_dataframe( - input_dir=".", + input_dir=tmp_dir, file_id=[["Test data", GRIB_FILENAME]], stats_file_name=STATS_FILE_NAMES, file_specification=file_specification, @@ -141,7 +141,6 @@ class TestStatsNetcdf(unittest.TestCase): nc_file_name = "test_stats.nc" nc_file_glob = "test_s*.nc" - stats_file_names = "test_stats.csv" def setUp(self): data = initialize_dummy_netcdf_file(self.nc_file_name) @@ -168,7 +167,7 @@ def setUp(self): def tear_down(self): os.remove(self.nc_file_name) - os.remove(self.stats_file_names) + os.remove(STATS_FILE_NAMES) def test_stats(self): file_specification = { @@ -183,7 +182,7 @@ def test_stats(self): df = create_stats_dataframe( input_dir=".", file_id=[["Test data", self.nc_file_glob]], - stats_file_name=self.stats_file_names, + stats_file_name=STATS_FILE_NAMES, file_specification=file_specification, ) From 7be1ac48314fde497885449d3da42f714447d5a4 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 3 Oct 2024 12:12:20 +0200 Subject: [PATCH 22/62] change to pytest --- tests/engine/test_stats.py | 120 ++++++++++++++++++------------------- 1 file changed, 58 insertions(+), 62 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index a9549d24..d20a2066 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -25,6 +25,8 @@ GRIB_FILENAME = "test_stats_grib.grib" STATS_FILE_NAMES = "test_stats.csv" +NC_FILE_NAME = "test_stats.nc" +NC_FILE_GLOB = "test_s*.nc" def initialize_dummy_netcdf_file(name): @@ -129,80 +131,74 @@ def test_stats_grib(tmp_dir): ), f"Stats dataframe incorrect. Difference:\n{df.values == expected}" -class TestStatsNetcdf(unittest.TestCase): - """ - Unit test class for validating statistical calculations from NetCDF files. +@pytest.fixture(name="setup_netcdf_file") +def fixture_setup_netcdf_file(tmp_dir): + """Fixture to create and initialize a dummy NetCDF file for testing.""" - This class tests the accuracy of statistical calculations (mean, max, min) - performed on data extracted from NetCDF files. - It ensures that the statistics DataFrame produced from the NetCDF data - matches expected values. - """ + data = initialize_dummy_netcdf_file(os.path.join(tmp_dir, NC_FILE_NAME)) - nc_file_name = "test_stats.nc" - nc_file_glob = "test_s*.nc" + # Creating variable "v1" with specified dimensions and setting its values + data.createVariable("v1", np.float64, dimensions=("t", "z", "x")) + data.variables["v1"][:] = np.ones((TIME_DIM_SIZE, HEIGHT_DIM_SIZE, HOR_DIM_SIZE)) + data.variables["v1"][:, :, 0] = 0 + data.variables["v1"][:, :, -1] = 2 - def setUp(self): - data = initialize_dummy_netcdf_file(self.nc_file_name) + # Creating variable "v2" with fill_value, and setting its values + data.createVariable("v2", np.float64, dimensions=("t", "x"), fill_value=42) + data.variables["v2"][:] = np.ones((TIME_DIM_SIZE, HOR_DIM_SIZE)) * 2 + data.variables["v2"][:, 0] = 1 + data.variables["v2"][:, 1] = 42 # should be ignored in max-statistic + data.variables["v2"][:, -1] = 3 - data.createVariable("v1", np.float64, dimensions=("t", "z", "x")) - data.variables["v1"][:] = np.ones( - (TIME_DIM_SIZE, HEIGHT_DIM_SIZE, HOR_DIM_SIZE) - ) - data.variables["v1"][:, :, 0] = 0 - data.variables["v1"][:, :, -1] = 2 + # Creating variable "v3" and setting its values + data.createVariable("v3", np.float64, dimensions=("t", "x")) + data.variables["v3"][:] = np.ones((TIME_DIM_SIZE, HOR_DIM_SIZE)) * 3 + data.variables["v3"][:, 0] = 2 + data.variables["v3"][:, -1] = 4 - data.createVariable("v2", np.float64, dimensions=("t", "x"), fill_value=42) - data.variables["v2"][:] = np.ones((TIME_DIM_SIZE, HOR_DIM_SIZE)) * 2 - data.variables["v2"][:, 0] = 1 - data.variables["v2"][:, 1] = 42 # shall be ignored in max-statistic - data.variables["v2"][:, -1] = 3 + data.close() - data.createVariable("v3", np.float64, dimensions=("t", "x")) - data.variables["v3"][:] = np.ones((TIME_DIM_SIZE, HOR_DIM_SIZE)) * 3 - data.variables["v3"][:, 0] = 2 - data.variables["v3"][:, -1] = 4 + yield - data.close() - def tear_down(self): - os.remove(self.nc_file_name) - os.remove(STATS_FILE_NAMES) +def test_stats_netcdf(setup_netcdf_file, tmp_dir): # pylint: disable=unused-argument + """Test that the statistics generated from the NetCDF file match the + expected values.""" - def test_stats(self): - file_specification = { - "Test data": { - "format": "netcdf", - "time_dim": "t", - "horizontal_dims": ["x"], - "fill_value_key": "_FillValue", # should be the name for fill_value - }, - } + file_specification = { + "Test data": { + "format": "netcdf", + "time_dim": "t", + "horizontal_dims": ["x"], + "fill_value_key": "_FillValue", # should be the name for fill_value + }, + } - df = create_stats_dataframe( - input_dir=".", - file_id=[["Test data", self.nc_file_glob]], - stats_file_name=STATS_FILE_NAMES, - file_specification=file_specification, - ) + # Call the function to generate the statistics dataframe + df = create_stats_dataframe( + input_dir=tmp_dir, + file_id=[["Test data", NC_FILE_GLOB]], + stats_file_name=STATS_FILE_NAMES, + file_specification=file_specification, + ) - # check that the mean/max/min are correct - expected = np.array( - [ - [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], - [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], - [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], - [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], - [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], - [2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0], - [3.0, 4.0, 2.0, 3.0, 4.0, 2.0, 3.0, 4.0, 2.0], - ] - ) + # Define the expected values for comparison + expected = np.array( + [ + [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], + [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], + [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], + [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], + [1.0, 2.0, 0.0, 1.0, 2.0, 0.0, 1.0, 2.0, 0.0], + [2.0, 3.0, 1.0, 2.0, 3.0, 1.0, 2.0, 3.0, 1.0], + [3.0, 4.0, 2.0, 3.0, 4.0, 2.0, 3.0, 4.0, 2.0], + ] + ) - self.assertTrue( - np.array_equal(df.values, expected), - f"stats dataframe incorrect. Difference:\n{df.values == expected}", - ) + # Check that the dataframe matches the expected values + assert np.array_equal( + df.values, expected + ), f"Stats dataframe incorrect. Difference:\n{df.values == expected}" class TestStatsCsv(unittest.TestCase): From 99320d85f6122d53b5e09a714c9e9fe3abd467a5 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 3 Oct 2024 12:48:53 +0200 Subject: [PATCH 23/62] cleanup tmp path --- .gitignore | 3 - tests/engine/test_stats.py | 127 +++++++++++++++++-------------------- 2 files changed, 57 insertions(+), 73 deletions(-) diff --git a/.gitignore b/.gitignore index 540f87eb..1ef0787d 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,4 @@ eccodes-cosmo-resources miniconda # output from unittest -test_stats.csv -test_stats_csv.csv -test_stats_csv.dat random_tolerance.csv diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index d20a2066..88bcb1cf 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -4,9 +4,6 @@ dataframes from both NetCDF and CSV files. """ -import os -import unittest - import eccodes import numpy as np import pytest @@ -67,7 +64,7 @@ def add_variable_to_grib(filename, dict_data): @pytest.fixture -def setup_grib_file(tmp_dir): +def setup_grib_file(tmp_path): array_t = np.ones( ( TIME_DIM_GRIB_SIZE, @@ -96,11 +93,11 @@ def setup_grib_file(tmp_dir): dict_data = {"t": array_pres, "v": array_t} # This would be where your grib file is created - add_variable_to_grib(os.path.join(tmp_dir, GRIB_FILENAME), dict_data) + add_variable_to_grib(tmp_path / GRIB_FILENAME, dict_data) @pytest.mark.usefixtures("setup_grib_file") -def test_stats_grib(tmp_dir): +def test_stats_grib(tmp_path): file_specification = { "Test data": { "format": "grib", @@ -112,7 +109,7 @@ def test_stats_grib(tmp_dir): } df = create_stats_dataframe( - input_dir=tmp_dir, + input_dir=str(tmp_path), file_id=[["Test data", GRIB_FILENAME]], stats_file_name=STATS_FILE_NAMES, file_specification=file_specification, @@ -132,10 +129,10 @@ def test_stats_grib(tmp_dir): @pytest.fixture(name="setup_netcdf_file") -def fixture_setup_netcdf_file(tmp_dir): +def fixture_setup_netcdf_file(tmp_path): """Fixture to create and initialize a dummy NetCDF file for testing.""" - data = initialize_dummy_netcdf_file(os.path.join(tmp_dir, NC_FILE_NAME)) + data = initialize_dummy_netcdf_file(tmp_path / NC_FILE_NAME) # Creating variable "v1" with specified dimensions and setting its values data.createVariable("v1", np.float64, dimensions=("t", "z", "x")) @@ -161,7 +158,7 @@ def fixture_setup_netcdf_file(tmp_dir): yield -def test_stats_netcdf(setup_netcdf_file, tmp_dir): # pylint: disable=unused-argument +def test_stats_netcdf(setup_netcdf_file, tmp_path): # pylint: disable=unused-argument """Test that the statistics generated from the NetCDF file match the expected values.""" @@ -176,7 +173,7 @@ def test_stats_netcdf(setup_netcdf_file, tmp_dir): # pylint: disable=unused-arg # Call the function to generate the statistics dataframe df = create_stats_dataframe( - input_dir=tmp_dir, + input_dir=str(tmp_path), file_id=[["Test data", NC_FILE_GLOB]], stats_file_name=STATS_FILE_NAMES, file_specification=file_specification, @@ -201,70 +198,60 @@ def test_stats_netcdf(setup_netcdf_file, tmp_dir): # pylint: disable=unused-arg ), f"Stats dataframe incorrect. Difference:\n{df.values == expected}" -class TestStatsCsv(unittest.TestCase): +@pytest.fixture(name="setup_csv_file") +def fixture_setup_csv_file(tmp_path): """ - Test suite for validating statistical calculations and CSV file handling. - - This class contains unit tests for creating and validating statistics from a - CSV file. - The primary focus is on ensuring that the statistics calculated from the - input data match the expected values. - The CSV file used for testing is created and cleaned up during the test - lifecycle. + Fixture to set up a temporary CSV file. """ + dat_file_name = tmp_path / "test_stats_csv.dat" + + # Create the CSV file with the necessary content + lines = ( + "time v1 v2 v3 v4 v5", + "10 1.4 15 16 17 18", + "20 2.4 25 26 27 28", + "30 3.4 35 36 37 38", + ) + with open(dat_file_name, "w", encoding="utf-8") as f: + f.write("\n".join(lines)) - dat_file_name = "test_stats_csv.dat" - stats_file_name = "test_stats_csv.csv" - - def setUp(self): - lines = ( - "time v1 v2 v3 v4 v5", - "10 1.4 15 16 17 18", - "20 2.4 25 26 27 28", - "30 3.4 35 36 37 38", - ) - with open(self.dat_file_name, "w", encoding="utf-8") as f: - f.write("\n".join(lines)) - - def tear_down(self): - os.remove(self.dat_file_name) - os.remove(self.stats_file_name) - - def test_stats(self): - file_specification = { - "Test data": { - "format": "csv", - "parser_args": { - "delimiter": "\\s+", - "header": 0, - "index_col": 0, - }, - }, - } - df = create_stats_dataframe( - input_dir=".", - file_id=[["Test data", self.dat_file_name]], - stats_file_name=self.stats_file_name, - file_specification=file_specification, - ) +def test_stats_csv(setup_csv_file, tmp_path): # pylint: disable=unused-argument + """ + Test that the statistics generated from the CSV file match the expected values. + """ - # check that the mean/max/min are correct (i.e. the same as in CSV) - expected = np.array( - [ - [1.4, 1.4, 1.4, 2.4, 2.4, 2.4, 3.4, 3.4, 3.4], - [15, 15, 15, 25, 25, 25, 35, 35, 35], - [16, 16, 16, 26, 26, 26, 36, 36, 36], - [17, 17, 17, 27, 27, 27, 37, 37, 37], - [18, 18, 18, 28, 28, 28, 38, 38, 38], - ], - ) + file_specification = { + "Test data": { + "format": "csv", + "parser_args": { + "delimiter": "\\s+", + "header": 0, + "index_col": 0, + }, + }, + } - self.assertTrue( - np.array_equal(df.values, expected), - f"stats dataframe incorrect. Difference:\n{df.values == expected}", - ) + # Call the function that creates the stats DataFrame + df = create_stats_dataframe( + input_dir=str(tmp_path), + file_id=[["Test data", "test_stats_csv.dat"]], + stats_file_name="test_stats_csv.csv", + file_specification=file_specification, + ) + # Expected result + expected = np.array( + [ + [1.4, 1.4, 1.4, 2.4, 2.4, 2.4, 3.4, 3.4, 3.4], + [15, 15, 15, 25, 25, 25, 35, 35, 35], + [16, 16, 16, 26, 26, 26, 36, 36, 36], + [17, 17, 17, 27, 27, 27, 37, 37, 37], + [18, 18, 18, 28, 28, 28, 38, 38, 38], + ] + ) -if __name__ == "__main__": - unittest.main() + # Assert the DataFrame matches the expected values + assert np.array_equal( + df.values, expected + ), f"Stats DataFrame incorrect. Difference:\n{df.values != expected}" From 92677deebf06bf505e60067d8f42d684290b6b07 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 3 Oct 2024 12:56:22 +0200 Subject: [PATCH 24/62] fix tmp --- tests/engine/test_stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 88bcb1cf..930fe20c 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -111,7 +111,7 @@ def test_stats_grib(tmp_path): df = create_stats_dataframe( input_dir=str(tmp_path), file_id=[["Test data", GRIB_FILENAME]], - stats_file_name=STATS_FILE_NAMES, + stats_file_name=tmp_path / STATS_FILE_NAMES, file_specification=file_specification, ) @@ -175,7 +175,7 @@ def test_stats_netcdf(setup_netcdf_file, tmp_path): # pylint: disable=unused-ar df = create_stats_dataframe( input_dir=str(tmp_path), file_id=[["Test data", NC_FILE_GLOB]], - stats_file_name=STATS_FILE_NAMES, + stats_file_name=tmp_path / STATS_FILE_NAMES, file_specification=file_specification, ) @@ -236,7 +236,7 @@ def test_stats_csv(setup_csv_file, tmp_path): # pylint: disable=unused-argument df = create_stats_dataframe( input_dir=str(tmp_path), file_id=[["Test data", "test_stats_csv.dat"]], - stats_file_name="test_stats_csv.csv", + stats_file_name=tmp_path / "test_stats_csv.csv", file_specification=file_specification, ) From 658c2498bd6e85d0784de10e8cf7ea2d5631963c Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 3 Oct 2024 13:20:29 +0200 Subject: [PATCH 25/62] cleanup gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 1ef0787d..c9450fa9 100644 --- a/.gitignore +++ b/.gitignore @@ -11,5 +11,6 @@ coverage.xml eccodes-cosmo-resources miniconda -# output from unittest +# output from pytest random_tolerance.csv +random_tolerance_None.csv From 32a180b3b7c931c3ec4b5965dca8d881f93217d7 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 09:23:13 +0200 Subject: [PATCH 26/62] fix utest --- tests/engine/test_stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 930fe20c..d086efec 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -118,8 +118,8 @@ def test_stats_grib(tmp_path): # check that the mean/max/min are correct expected = np.array( [ - [3.0, 4.0, 2.0], [1.0, 2.0, 0.0], + [3.0, 4.0, 2.0], ] ) From c16e2218e890afa2b82e6f7c5bb9127f64186098 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 09:23:42 +0200 Subject: [PATCH 27/62] fix env --- setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_env.sh b/setup_env.sh index ccaeab0a..dc802eee 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -73,7 +73,7 @@ fi # setting ECCODES_DEFINITION_PATH: git clone --recursive https://github.com/COSMO-ORG/eccodes-cosmo-resources.git -${CONDA} activate ${DEV_ENV_NAME} +${CONDA} activate ${ENV_NAME} conda_loc=${CONDA_PREFIX} base_dir=$(pwd) def_path_default=${conda_loc}/share/eccodes/definitions From d2d84d366b3be329919a3d0ccd5b0c3adcc3d1fe Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 09:48:27 +0200 Subject: [PATCH 28/62] improve github actions --- .github/workflows/pytest.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index b91e66bd..90cb69b7 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -26,11 +26,11 @@ jobs: channels: conda-forge channel-priority: flexible show-channel-urls: true - - name: Create dev env from pinned reqs + - name: Create env from pinned reqs run: | - conda env create --name dev_env --file requirements/environment.yml + ./setup_env.sh -n probtest - name: Run Pytest env: TZ: Europe/Zurich run: | - conda run --name dev_env pytest -v -s --cov --cov-report=term tests/ + conda run --name env pytest -v -s --cov --cov-report=term tests/ From 4f7e183fd089ec75d5c1c372c1f75c12beec1e85 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 09:53:13 +0200 Subject: [PATCH 29/62] fix env --- .github/workflows/pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 90cb69b7..2fc2caa4 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -33,4 +33,4 @@ jobs: env: TZ: Europe/Zurich run: | - conda run --name env pytest -v -s --cov --cov-report=term tests/ + conda run --name probtest pytest -v -s --cov --cov-report=term tests/ From 74ad3014aa6488997350390ca520d2dd50efe317 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:12:25 +0200 Subject: [PATCH 30/62] step by step refactor --- requirements/environment.yml | 29 ++++++++++++++++++----------- setup_env.sh | 17 +++++++++-------- 2 files changed, 27 insertions(+), 19 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 6516b500..abcded61 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -50,6 +50,7 @@ dependencies: - colorama=0.4.6 - contourpy=1.3.0 - cycler=0.12.1 + - cyrus-sasl=2.1.27 - cytoolz=0.12.3 - dask=2024.9.1 - dask-core=2024.9.1 @@ -100,7 +101,7 @@ dependencies: - importlib_resources=6.4.5 - iniconfig=2.0.0 - ipdb=0.13.13 - - ipython=8.27.0 + - ipython=8.28.0 - isort=5.13.2 - jasper=4.2.4 - jedi=0.19.1 @@ -154,16 +155,17 @@ dependencies: - libiconv=1.17 - libjpeg-turbo=3.0.0 - liblapack=3.9.0 - - libllvm19=19.1.0 + - libllvm19=19.1.1 - libnetcdf=4.9.2 - libnghttp2=1.58.0 - libnsl=2.0.1 + - libntlm=1.4 - libopenblas=0.3.27 - libopengl=1.7.0 - libparquet=17.0.0 - libpciaccess=0.18 - libpng=1.6.44 - - libpq=16.4 + - libpq=17.0 - libprotobuf=5.27.5 - libre2-11=2023.09.01 - libsqlite=3.46.1 @@ -205,6 +207,7 @@ dependencies: - nodeenv=1.9.1 - numpy=2.1.1 - openjpeg=2.5.2 + - openldap=2.6.8 - openssl=3.3.2 - orc=2.0.2 - packaging=24.1 @@ -242,7 +245,7 @@ dependencies: - pygments=2.18.0 - pylint=3.3.1 - pyparsing=3.1.4 - - pyside6=6.7.2 + - pyside6=6.7.3 - pysocks=1.7.1 - pytest=8.3.3 - python=3.12.6 @@ -254,13 +257,13 @@ dependencies: - pytz=2024.1 - pyyaml=6.0.2 - qhull=2020.2 - - qt6-main=6.7.2 + - qt6-main=6.7.3 - re2=2023.09.01 - readline=8.2 - referencing=0.35.1 - regex=2024.9.11 - requests=2.32.3 - - rich=13.8.1 + - rich=13.9.1 - rope=1.13.0 - rpds-py=0.20.0 - rstcheck=6.2.4 @@ -279,7 +282,7 @@ dependencies: - tblib=3.0.0 - tk=8.6.13 - toml=0.10.2 - - tomli=2.0.1 + - tomli=2.0.2 - tomlkit=0.13.2 - toolz=0.12.1 - tornado=6.4.1 @@ -304,15 +307,19 @@ dependencies: - xcb-util-keysyms=0.4.1 - xcb-util-renderutil=0.3.10 - xcb-util-wm=0.4.2 - - xkeyboard-config=2.42 + - xkeyboard-config=2.43 - xorg-libice=1.1.1 - xorg-libsm=1.2.4 - xorg-libx11=1.8.10 - xorg-libxau=1.0.11 - - xorg-libxdmcp=1.1.3 + - xorg-libxcomposite=0.4.6 + - xorg-libxcursor=1.2.2 + - xorg-libxdamage=1.1.6 + - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - - xorg-libxfixes=6.0.0 - - xorg-libxi=1.7.10 + - xorg-libxfixes=6.0.1 + - xorg-libxi=1.8.2 + - xorg-libxrandr=1.5.4 - xorg-libxrender=0.9.11 - xorg-libxtst=1.2.5 - xorg-libxxf86vm=1.1.5 diff --git a/setup_env.sh b/setup_env.sh index dc802eee..96b0db35 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -70,14 +70,15 @@ else fi -# setting ECCODES_DEFINITION_PATH: -git clone --recursive https://github.com/COSMO-ORG/eccodes-cosmo-resources.git - +# Setting ECCODES_DEFINITION_PATH: ${CONDA} activate ${ENV_NAME} -conda_loc=${CONDA_PREFIX} -base_dir=$(pwd) -def_path_default=${conda_loc}/share/eccodes/definitions -def_path_resources=${base_dir}/eccodes-cosmo-resources/definitions -conda env config vars set ECCODES_DEFINITION_PATH=${def_path_default}:${def_path_resources} +git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git + +CONDA_LOC=${CONDA_PREFIX} +BASE_DIR=$(pwd) +DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes/definitions +DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources/definitions + +conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}:${DEF_PATH_RESOURCES} ${CONDA} deactivate From 3996217244d274e36642f8caec388aba35887fb2 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:20:49 +0200 Subject: [PATCH 31/62] next step --- setup_env.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 96b0db35..bf9b76aa 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -73,12 +73,13 @@ fi # Setting ECCODES_DEFINITION_PATH: ${CONDA} activate ${ENV_NAME} -git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git - CONDA_LOC=${CONDA_PREFIX} BASE_DIR=$(pwd) -DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes/definitions -DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources/definitions +DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes +DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources + +git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git + +conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions -conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}:${DEF_PATH_RESOURCES} ${CONDA} deactivate From 3b467e5a33b90677964119fb727c9f6bf27bf05c Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:25:20 +0200 Subject: [PATCH 32/62] next step --- setup_env.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/setup_env.sh b/setup_env.sh index bf9b76aa..2333a132 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -80,6 +80,9 @@ DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git -conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions +conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions || exit + +echo "Variables saved to environment: " +${CONDA} env config vars list ${CONDA} deactivate From 30bbcd3b7b2f3826d1261f90808c25e4513cd318 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:29:19 +0200 Subject: [PATCH 33/62] next step --- setup_env.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 2333a132..256af8b7 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -78,9 +78,9 @@ BASE_DIR=$(pwd) DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources -git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git +git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${DEF_PATH_RESOURCES} || exit -conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions || exit +conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions echo "Variables saved to environment: " ${CONDA} env config vars list From bc0be679a18d68029e25417077a928c1c78ff273 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:33:43 +0200 Subject: [PATCH 34/62] last step --- setup_env.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 256af8b7..4eb6231b 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -74,9 +74,8 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -BASE_DIR=$(pwd) DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes -DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources +DEF_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${DEF_PATH_RESOURCES} || exit From b5c28663fe0fef26a4f0b639ef72cb6b5de622b0 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:33:43 +0200 Subject: [PATCH 35/62] last step --- setup_env.sh | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 256af8b7..96932532 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -74,13 +74,12 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -BASE_DIR=$(pwd) DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes -DEF_PATH_RESOURCES=${BASE_DIR}/eccodes-cosmo-resources +DEF_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${DEF_PATH_RESOURCES} || exit -conda env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions +${CONDA} env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions echo "Variables saved to environment: " ${CONDA} env config vars list From ad6e44e7cc351b8d6626b0644ede4725fa4c0966 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:38:13 +0200 Subject: [PATCH 36/62] cleanup gitignore --- .gitignore | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitignore b/.gitignore index c9450fa9..741ab569 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,6 @@ *.log *.nc coverage.xml -eccodes-cosmo-resources miniconda # output from pytest From 8b074d7a99d1b6b6d22a4115e58f87e6a7958ce1 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:39:38 +0200 Subject: [PATCH 37/62] minor change --- setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_env.sh b/setup_env.sh index 96932532..f2ce71e0 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -13,7 +13,7 @@ DEFAULT_ENV_NAME="probtest" # Default options ENV_NAME="${DEFAULT_ENV_NAME}" -PYVERSION=3.12.6 +PYVERSION=3.12 PINNED=true EXPORT=false CONDA=conda From 7e1427bd2ec9d2b22fff772122181a16d66b3a48 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:48:08 +0200 Subject: [PATCH 38/62] fix readme --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 721ba838..ad18150a 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ setup scripts: ```console ./setup_miniconda.sh +source miniconda/bin/activate ./setup_env.sh -n probtest -u ``` From 3f9bd0d2f19736472f88737c1f19792138dee06f Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 10:54:54 +0200 Subject: [PATCH 39/62] fix definitions version --- setup_env.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index f2ce71e0..552190e2 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -74,12 +74,13 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEF_PATH_DEFAULT=${CONDA_LOC}/share/eccodes -DEF_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources +DEFINITION_VERSION="v2.25.0.3" +DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes +DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} -git clone https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${DEF_PATH_RESOURCES} || exit +git clone -b ${DEFINITION_VERSION} https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${DEFINITION_PATH_RESOURCES} || exit -${CONDA} env config vars set ECCODES_DEFINITION_PATH=${DEF_PATH_DEFAULT}/definitions:${DEF_PATH_RESOURCES}/definitions +${CONDA} env config vars set ECCODES_DEFINITION_PATH=${DEFINITION_PATH_DEFAULT}/definitions:${DEFINITION_PATH_RESOURCES}/definitions echo "Variables saved to environment: " ${CONDA} env config vars list From 8f89c852e240bc525b4b5b1944149668e187298c Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 11:26:44 +0200 Subject: [PATCH 40/62] make icon consistent --- setup_env.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup_env.sh b/setup_env.sh index 552190e2..c04c6f2c 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -74,7 +74,7 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEFINITION_VERSION="v2.25.0.3" +DEFINITION_VERSION="v2.25.0.2" DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} From 6039666ffce695f2d9d24942f283b8c13eaf2f25 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 13:25:31 +0200 Subject: [PATCH 41/62] fix naming --- tests/engine/test_perturb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/engine/test_perturb.py b/tests/engine/test_perturb.py index f78b5c69..a420bbe6 100644 --- a/tests/engine/test_perturb.py +++ b/tests/engine/test_perturb.py @@ -12,8 +12,8 @@ from engine.perturb import perturb_array -atype = np.float32 -AMPLITUDE = atype(1e-14) +ATYPE = np.float32 +AMPLITUDE = ATYPE(1e-14) ARRAY_DIM = 100 @@ -33,8 +33,8 @@ def fixture_create_nc_files(tmp_dir): def test_perturb_array(): # create two arrays, perturb one. - x1 = np.ones((ARRAY_DIM, ARRAY_DIM), dtype=atype) - x2 = np.ones((ARRAY_DIM, ARRAY_DIM), dtype=atype) + x1 = np.ones((ARRAY_DIM, ARRAY_DIM), dtype=ATYPE) + x2 = np.ones((ARRAY_DIM, ARRAY_DIM), dtype=ATYPE) x_perturbed = perturb_array(x2, 10, AMPLITUDE) # compute some stats and do assertions From 7bfcea53e0d6de3d2330b91609447a0f6ae58895 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 13:25:53 +0200 Subject: [PATCH 42/62] fix naming --- tests/engine/test_stats.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index d086efec..e35b1a8c 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -47,9 +47,7 @@ def initialize_dummy_netcdf_file(name): def add_variable_to_grib(filename, dict_data): with open(filename, "wb") as f_out: for short_name in list(dict_data.keys()): - gid = eccodes.codes_grib_new_from_samples( - "reduced_rotated_gg_sfc_grib2.tmpl" - ) + gid = eccodes.codes_grib_new_from_samples("reduced_rotated_gg_sfc_grib2") eccodes.codes_set(gid, "edition", 2) eccodes.codes_set(gid, "centre", "lssw") eccodes.codes_set(gid, "dataDate", 20230913) From 903f8a456a9424060c331755457653c5e6340411 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 13:26:23 +0200 Subject: [PATCH 43/62] fix eccodes version --- requirements/environment.yml | 195 +++++++++++++--------------------- requirements/requirements.yml | 3 +- 2 files changed, 77 insertions(+), 121 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index abcded61..79367495 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -5,31 +5,24 @@ channels: dependencies: - _libgcc_mutex=0.1 - _openmp_mutex=4.5 - - alsa-lib=1.2.12 - annotated-types=0.7.0 - astroid=3.3.4 - asttokens=2.4.1 - attrs=24.2.0 - - aws-c-auth=0.7.31 - - aws-c-cal=0.7.4 - - aws-c-common=0.9.28 - - aws-c-compression=0.2.19 - - aws-c-event-stream=0.4.3 - - aws-c-http=0.8.10 - - aws-c-io=0.14.18 - - aws-c-mqtt=0.10.6 - - aws-c-s3=0.6.6 - - aws-c-sdkutils=0.1.19 - - aws-checksums=0.1.20 - - aws-crt-cpp=0.28.3 - - aws-sdk-cpp=1.11.407 - - azure-core-cpp=1.13.0 - - azure-identity-cpp=1.9.0 - - azure-storage-blobs-cpp=12.13.0 - - azure-storage-common-cpp=12.8.0 - - azure-storage-files-datalake-cpp=12.12.0 + - aws-c-auth=0.7.22 + - aws-c-cal=0.6.14 + - aws-c-common=0.9.19 + - aws-c-compression=0.2.18 + - aws-c-event-stream=0.4.2 + - aws-c-http=0.8.1 + - aws-c-io=0.14.8 + - aws-c-mqtt=0.10.4 + - aws-c-s3=0.5.9 + - aws-c-sdkutils=0.1.16 + - aws-checksums=0.1.18 + - aws-crt-cpp=0.26.9 + - aws-sdk-cpp=1.11.329 - black=24.8.0 - - blosc=1.21.6 - bokeh=3.5.2 - brotli=1.1.0 - brotli-bin=1.1.0 @@ -37,7 +30,6 @@ dependencies: - bzip2=1.0.8 - c-ares=1.33.1 - ca-certificates=2024.8.30 - - cairo=1.18.0 - certifi=2024.8.30 - cffi=1.17.1 - cfgrib=0.9.14.1 @@ -49,22 +41,20 @@ dependencies: - codespell=2.3.0 - colorama=0.4.6 - contourpy=1.3.0 + - curl=8.9.1 - cycler=0.12.1 - - cyrus-sasl=2.1.27 - cytoolz=0.12.3 - dask=2024.9.1 - dask-core=2024.9.1 - dask-expr=1.1.15 - - dbus=1.13.6 + - dbus=1.13.18 - decorator=5.1.1 - dill=0.3.9 - distlib=0.3.8 - distributed=2024.9.1 - docutils=0.21.2 - - double-conversion=3.3.0 - - earthkit-data=0.10.4 - - earthkit-geo=0.2.0 - - eccodes=2.38.0 + - earthkit-data=0.5.6 + - eccodes=2.25.0 - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 @@ -73,67 +63,59 @@ dependencies: - findlibs=0.0.5 - flake8=7.1.1 - flake8-black=0.3.6 - - font-ttf-dejavu-sans-mono=2.37 - - font-ttf-inconsolata=3.000 - - font-ttf-source-code-pro=2.038 - - font-ttf-ubuntu=0.83 - fontconfig=2.14.2 - - fonts-conda-ecosystem=1 - - fonts-conda-forge=1 - fonttools=4.54.1 - freeglut=3.2.2 - freetype=2.12.1 - fsspec=2024.9.0 - gflags=2.2.2 + - glib=2.80.2 + - glib-tools=2.80.2 - glog=0.7.1 - - graphite2=1.3.13 + - gst-plugins-base=1.14.1 + - gstreamer=1.14.1 - h2=4.1.0 - - harfbuzz=9.0.0 - hdf4=4.2.15 - - hdf5=1.14.3 + - hdf5=1.12.1 - hpack=4.0.0 - hyperframe=6.0.1 - - icu=75.1 + - icu=73.2 - identify=2.6.1 - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - - importlib_resources=6.4.5 - iniconfig=2.0.0 - ipdb=0.13.13 - ipython=8.28.0 - isort=5.13.2 - - jasper=4.2.4 + - jasper=2.0.33 - jedi=0.19.1 - jinja2=3.1.4 - - jsonschema=4.23.0 - - jsonschema-specifications=2023.12.1 + - jpeg=9e - keyutils=1.6.1 - kiwisolver=1.4.7 - - krb5=1.21.3 - - lcms2=2.16 + - krb5=1.20.1 + - lcms2=2.12 - ld_impl_linux-64=2.43 - - lerc=4.0.0 - - libabseil=20240722.0 + - lerc=3.0 + - libabseil=20240116.2 - libaec=1.1.3 - - libarrow=17.0.0 - - libarrow-acero=17.0.0 - - libarrow-dataset=17.0.0 - - libarrow-substrait=17.0.0 + - libarrow=16.1.0 + - libarrow-acero=16.1.0 + - libarrow-dataset=16.1.0 + - libarrow-substrait=16.1.0 - libblas=3.9.0 - libbrotlicommon=1.1.0 - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 - - libclang-cpp19.1=19.1.0 - - libclang13=19.1.0 + - libclang=14.0.6 + - libclang13=14.0.6 - libcrc32c=1.1.2 - libcups=2.3.3 - - libcurl=8.10.1 - - libdeflate=1.21 - - libdrm=2.4.123 + - libcurl=8.9.1 + - libdeflate=1.10 - libedit=3.1.20191231 - - libegl=1.7.0 - libev=4.33 - libevent=2.1.12 - libexpat=2.6.3 @@ -143,37 +125,30 @@ dependencies: - libgfortran=14.1.0 - libgfortran-ng=14.1.0 - libgfortran5=14.1.0 - - libgl=1.7.0 - - libglib=2.82.1 + - libglib=2.80.2 - libglu=9.0.0 - - libglvnd=1.7.0 - - libglx=1.7.0 - libgomp=14.1.0 - - libgoogle-cloud=2.29.0 - - libgoogle-cloud-storage=2.29.0 - - libgrpc=1.65.5 + - libgoogle-cloud=2.24.0 + - libgoogle-cloud-storage=2.24.0 + - libgrpc=1.62.2 - libiconv=1.17 - - libjpeg-turbo=3.0.0 - liblapack=3.9.0 - - libllvm19=19.1.1 - - libnetcdf=4.9.2 + - libllvm14=14.0.6 + - libnetcdf=4.8.1 - libnghttp2=1.58.0 - libnsl=2.0.1 - - libntlm=1.4 - libopenblas=0.3.27 - - libopengl=1.7.0 - - libparquet=17.0.0 - - libpciaccess=0.18 - - libpng=1.6.44 - - libpq=17.0 - - libprotobuf=5.27.5 + - libparquet=16.1.0 + - libpng=1.6.43 + - libpq=12.17 + - libprotobuf=4.25.3 - libre2-11=2023.09.01 - - libsqlite=3.46.1 + - libsqlite=3.46.0 - libssh2=1.11.0 - libstdcxx=14.1.0 - libstdcxx-ng=14.1.0 - - libthrift=0.21.0 - - libtiff=4.7.0 + - libthrift=0.19.0 + - libtiff=4.3.0 - libutf8proc=2.8.0 - libuuid=2.38.1 - libwebp-base=1.4.0 @@ -181,18 +156,15 @@ dependencies: - libxcrypt=4.4.36 - libxkbcommon=1.7.0 - libxml2=2.12.7 - - libxslt=1.1.39 - - libzip=1.11.1 - - libzlib=1.3.1 + - libzip=1.10.1 + - libzlib=1.2.13 - locket=1.0.0 - - lru-dict=1.3.0 - lz4=4.3.3 - lz4-c=1.9.4 - - markdown=3.6 - markdown-it-py=3.0.0 - markupsafe=2.1.5 - - matplotlib=3.9.2 - - matplotlib-base=3.9.2 + - matplotlib=3.9.1 + - matplotlib-base=3.9.1 - matplotlib-inline=0.1.7 - mccabe=0.7.0 - mdurl=0.1.2 @@ -200,30 +172,27 @@ dependencies: - munkres=1.1.4 - mypy=1.11.2 - mypy_extensions=1.0.0 - - mysql-common=9.0.1 - - mysql-libs=9.0.1 + - mysql=5.7.20 - ncurses=6.5 - - netcdf4=1.7.1 + - netcdf4=1.6.0 - nodeenv=1.9.1 - - numpy=2.1.1 - - openjpeg=2.5.2 - - openldap=2.6.8 + - numpy=1.26.4 + - olefile=0.47 + - openjpeg=2.5.0 - openssl=3.3.2 - - orc=2.0.2 + - orc=2.0.1 - packaging=24.1 - pandas=2.2.3 - parso=0.8.4 - partd=1.4.2 - pathlib=1.0.1 - pathspec=0.12.1 - - pcre2=10.44 + - pcre2=10.43 - pdbufr=0.11.0 - pexpect=4.9.0 - pickleshare=0.7.5 - - pillow=10.4.0 + - pillow=8.4.0 - pip=24.2 - - pixman=0.43.2 - - pkgutil-resolve-name=1.3.10 - platformdirs=4.3.6 - pluggy=1.5.0 - pre-commit=3.8.0 @@ -233,8 +202,8 @@ dependencies: - pthread-stubs=0.4 - ptyprocess=0.7.0 - pure_eval=0.2.3 - - pyarrow=17.0.0 - - pyarrow-core=17.0.0 + - pyarrow=16.1.0 + - pyarrow-core=16.1.0 - pyarrow-hotfix=0.6 - pycodestyle=2.12.1 - pycparser=2.22 @@ -245,39 +214,40 @@ dependencies: - pygments=2.18.0 - pylint=3.3.1 - pyparsing=3.1.4 - - pyside6=6.7.3 + - pyqt=5.15.4 + - pyqt5-sip=12.9.0 - pysocks=1.7.1 - pytest=8.3.3 - - python=3.12.6 + - python=3.10.14 - python-dateutil=2.9.0 - - python-eccodes=2.37.0 + - python-eccodes=1.4.0 - python-tzdata=2024.2 - - python_abi=3.12 + - python_abi=3.10 - pytoolconfig=1.2.5 - pytz=2024.1 - pyyaml=6.0.2 - qhull=2020.2 - - qt6-main=6.7.3 + - qt-main=5.15.2 - re2=2023.09.01 - readline=8.2 - - referencing=0.35.1 - regex=2024.9.11 - requests=2.32.3 - rich=13.9.1 - rope=1.13.0 - - rpds-py=0.20.0 - rstcheck=6.2.4 - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 - ruamel.yaml.clib=0.2.8 - - s2n=1.5.3 + - s2n=1.4.15 - scipy=1.14.1 - setuptools=75.1.0 - shellingham=1.5.4 + - sip=6.5.1 - six=1.16.0 - snappy=1.2.1 - snowballstemmer=2.2.0 - sortedcontainers=2.4.0 + - sqlite=3.46.0 - stack_data=0.6.2 - tblib=3.0.0 - tk=8.6.13 @@ -295,34 +265,19 @@ dependencies: - typing_extensions=4.12.2 - tzdata=2024a - ukkonen=1.0.1 + - unicodedata2=15.1.0 - urllib3=2.2.3 - virtualenv=20.26.6 - - wayland=1.23.1 - wcwidth=0.2.13 - wheel=0.44.0 - xarray=2024.9.0 - - xcb-util=0.4.1 - - xcb-util-cursor=0.1.5 - - xcb-util-image=0.4.0 - - xcb-util-keysyms=0.4.1 - - xcb-util-renderutil=0.3.10 - - xcb-util-wm=0.4.2 - xkeyboard-config=2.43 - - xorg-libice=1.1.1 - - xorg-libsm=1.2.4 - xorg-libx11=1.8.10 - xorg-libxau=1.0.11 - - xorg-libxcomposite=0.4.6 - - xorg-libxcursor=1.2.2 - - xorg-libxdamage=1.1.6 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxfixes=6.0.1 - xorg-libxi=1.8.2 - - xorg-libxrandr=1.5.4 - - xorg-libxrender=0.9.11 - - xorg-libxtst=1.2.5 - - xorg-libxxf86vm=1.1.5 - xorg-xextproto=7.3.0 - xorg-xorgproto=2024.1 - xyzservices=2024.9.0 @@ -330,7 +285,7 @@ dependencies: - yaml=0.2.5 - zict=3.0.0 - zipp=3.20.2 - - zlib=1.3.1 + - zlib=1.2.13 - zstandard=0.23.0 - zstd=1.5.6 - pip: diff --git a/requirements/requirements.yml b/requirements/requirements.yml index fb6bbf24..09ae938c 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -6,7 +6,8 @@ dependencies: - pip>=22.3 # runtime - click>=7.1.2 - - earthkit-data>=0.10.4 + - earthkit-data + - eccodes=2.25.0 - Jinja2>=3.0.1 - matplotlib>=3.2.1 - netCDF4>=1.5.3 From d9a1797894245db00450ac3385e740607d5ad37f Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 13:26:46 +0200 Subject: [PATCH 44/62] inlcude sample path --- setup_env.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/setup_env.sh b/setup_env.sh index c04c6f2c..898e3579 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -81,6 +81,9 @@ DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITIO git clone -b ${DEFINITION_VERSION} https://github.com/COSMO-ORG/eccodes-cosmo-resources.git ${DEFINITION_PATH_RESOURCES} || exit ${CONDA} env config vars set ECCODES_DEFINITION_PATH=${DEFINITION_PATH_DEFAULT}/definitions:${DEFINITION_PATH_RESOURCES}/definitions +${CONDA} env config vars set ECCODES_SAMPLES_PATH=${DEFINITION_PATH_DEFAULT}/samples +${CONDA} env config vars set GRIB_DEFINITION_PATH=${DEFINITION_PATH_DEFAULT}/definitions:${DEFINITION_PATH_RESOURCES}/definitions +${CONDA} env config vars set GRIB_SAMPLES_PATH=${DEFINITION_PATH_DEFAULT}/samples echo "Variables saved to environment: " ${CONDA} env config vars list From a584fcacc1132c8497bc1a934247cb67c7cac2f1 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 4 Oct 2024 15:04:32 +0200 Subject: [PATCH 45/62] use eccodes=2.35.0 --- requirements/environment.yml | 195 +++++++++++++++++++++------------- requirements/requirements.yml | 2 +- setup_env.sh | 2 +- 3 files changed, 122 insertions(+), 77 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 79367495..a796f19c 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -5,24 +5,31 @@ channels: dependencies: - _libgcc_mutex=0.1 - _openmp_mutex=4.5 + - alsa-lib=1.2.12 - annotated-types=0.7.0 - astroid=3.3.4 - asttokens=2.4.1 - attrs=24.2.0 - - aws-c-auth=0.7.22 - - aws-c-cal=0.6.14 - - aws-c-common=0.9.19 - - aws-c-compression=0.2.18 - - aws-c-event-stream=0.4.2 - - aws-c-http=0.8.1 - - aws-c-io=0.14.8 - - aws-c-mqtt=0.10.4 - - aws-c-s3=0.5.9 - - aws-c-sdkutils=0.1.16 - - aws-checksums=0.1.18 - - aws-crt-cpp=0.26.9 - - aws-sdk-cpp=1.11.329 + - aws-c-auth=0.7.31 + - aws-c-cal=0.7.4 + - aws-c-common=0.9.28 + - aws-c-compression=0.2.19 + - aws-c-event-stream=0.4.3 + - aws-c-http=0.8.10 + - aws-c-io=0.14.18 + - aws-c-mqtt=0.10.6 + - aws-c-s3=0.6.6 + - aws-c-sdkutils=0.1.19 + - aws-checksums=0.1.20 + - aws-crt-cpp=0.28.3 + - aws-sdk-cpp=1.11.407 + - azure-core-cpp=1.13.0 + - azure-identity-cpp=1.9.0 + - azure-storage-blobs-cpp=12.13.0 + - azure-storage-common-cpp=12.8.0 + - azure-storage-files-datalake-cpp=12.12.0 - black=24.8.0 + - blosc=1.21.6 - bokeh=3.5.2 - brotli=1.1.0 - brotli-bin=1.1.0 @@ -30,6 +37,7 @@ dependencies: - bzip2=1.0.8 - c-ares=1.33.1 - ca-certificates=2024.8.30 + - cairo=1.18.0 - certifi=2024.8.30 - cffi=1.17.1 - cfgrib=0.9.14.1 @@ -41,20 +49,22 @@ dependencies: - codespell=2.3.0 - colorama=0.4.6 - contourpy=1.3.0 - - curl=8.9.1 - cycler=0.12.1 + - cyrus-sasl=2.1.27 - cytoolz=0.12.3 - dask=2024.9.1 - dask-core=2024.9.1 - dask-expr=1.1.15 - - dbus=1.13.18 + - dbus=1.13.6 - decorator=5.1.1 - dill=0.3.9 - distlib=0.3.8 - distributed=2024.9.1 - docutils=0.21.2 - - earthkit-data=0.5.6 - - eccodes=2.25.0 + - double-conversion=3.3.0 + - earthkit-data=0.10.4 + - earthkit-geo=0.2.0 + - eccodes=2.35.0 - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 @@ -63,59 +73,67 @@ dependencies: - findlibs=0.0.5 - flake8=7.1.1 - flake8-black=0.3.6 + - font-ttf-dejavu-sans-mono=2.37 + - font-ttf-inconsolata=3.000 + - font-ttf-source-code-pro=2.038 + - font-ttf-ubuntu=0.83 - fontconfig=2.14.2 + - fonts-conda-ecosystem=1 + - fonts-conda-forge=1 - fonttools=4.54.1 - freeglut=3.2.2 - freetype=2.12.1 - fsspec=2024.9.0 - gflags=2.2.2 - - glib=2.80.2 - - glib-tools=2.80.2 - glog=0.7.1 - - gst-plugins-base=1.14.1 - - gstreamer=1.14.1 + - graphite2=1.3.13 - h2=4.1.0 + - harfbuzz=9.0.0 - hdf4=4.2.15 - - hdf5=1.12.1 + - hdf5=1.14.3 - hpack=4.0.0 - hyperframe=6.0.1 - - icu=73.2 + - icu=75.1 - identify=2.6.1 - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 + - importlib_resources=6.4.5 - iniconfig=2.0.0 - ipdb=0.13.13 - ipython=8.28.0 - isort=5.13.2 - - jasper=2.0.33 + - jasper=4.2.4 - jedi=0.19.1 - jinja2=3.1.4 - - jpeg=9e + - jsonschema=4.23.0 + - jsonschema-specifications=2023.12.1 - keyutils=1.6.1 - kiwisolver=1.4.7 - - krb5=1.20.1 - - lcms2=2.12 + - krb5=1.21.3 + - lcms2=2.16 - ld_impl_linux-64=2.43 - - lerc=3.0 - - libabseil=20240116.2 + - lerc=4.0.0 + - libabseil=20240722.0 - libaec=1.1.3 - - libarrow=16.1.0 - - libarrow-acero=16.1.0 - - libarrow-dataset=16.1.0 - - libarrow-substrait=16.1.0 + - libarrow=17.0.0 + - libarrow-acero=17.0.0 + - libarrow-dataset=17.0.0 + - libarrow-substrait=17.0.0 - libblas=3.9.0 - libbrotlicommon=1.1.0 - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 - - libclang=14.0.6 - - libclang13=14.0.6 + - libclang-cpp19.1=19.1.0 + - libclang13=19.1.0 - libcrc32c=1.1.2 - libcups=2.3.3 - - libcurl=8.9.1 - - libdeflate=1.10 + - libcurl=8.10.1 + - libdeflate=1.21 + - libdrm=2.4.123 - libedit=3.1.20191231 + - libegl=1.7.0 - libev=4.33 - libevent=2.1.12 - libexpat=2.6.3 @@ -125,30 +143,37 @@ dependencies: - libgfortran=14.1.0 - libgfortran-ng=14.1.0 - libgfortran5=14.1.0 - - libglib=2.80.2 + - libgl=1.7.0 + - libglib=2.82.1 - libglu=9.0.0 + - libglvnd=1.7.0 + - libglx=1.7.0 - libgomp=14.1.0 - - libgoogle-cloud=2.24.0 - - libgoogle-cloud-storage=2.24.0 - - libgrpc=1.62.2 + - libgoogle-cloud=2.29.0 + - libgoogle-cloud-storage=2.29.0 + - libgrpc=1.65.5 - libiconv=1.17 + - libjpeg-turbo=3.0.0 - liblapack=3.9.0 - - libllvm14=14.0.6 - - libnetcdf=4.8.1 + - libllvm19=19.1.1 + - libnetcdf=4.9.2 - libnghttp2=1.58.0 - libnsl=2.0.1 + - libntlm=1.4 - libopenblas=0.3.27 - - libparquet=16.1.0 - - libpng=1.6.43 - - libpq=12.17 - - libprotobuf=4.25.3 + - libopengl=1.7.0 + - libparquet=17.0.0 + - libpciaccess=0.18 + - libpng=1.6.44 + - libpq=17.0 + - libprotobuf=5.27.5 - libre2-11=2023.09.01 - - libsqlite=3.46.0 + - libsqlite=3.46.1 - libssh2=1.11.0 - libstdcxx=14.1.0 - libstdcxx-ng=14.1.0 - - libthrift=0.19.0 - - libtiff=4.3.0 + - libthrift=0.21.0 + - libtiff=4.7.0 - libutf8proc=2.8.0 - libuuid=2.38.1 - libwebp-base=1.4.0 @@ -156,15 +181,18 @@ dependencies: - libxcrypt=4.4.36 - libxkbcommon=1.7.0 - libxml2=2.12.7 - - libzip=1.10.1 - - libzlib=1.2.13 + - libxslt=1.1.39 + - libzip=1.11.1 + - libzlib=1.3.1 - locket=1.0.0 + - lru-dict=1.3.0 - lz4=4.3.3 - lz4-c=1.9.4 + - markdown=3.6 - markdown-it-py=3.0.0 - markupsafe=2.1.5 - - matplotlib=3.9.1 - - matplotlib-base=3.9.1 + - matplotlib=3.9.2 + - matplotlib-base=3.9.2 - matplotlib-inline=0.1.7 - mccabe=0.7.0 - mdurl=0.1.2 @@ -172,27 +200,30 @@ dependencies: - munkres=1.1.4 - mypy=1.11.2 - mypy_extensions=1.0.0 - - mysql=5.7.20 + - mysql-common=9.0.1 + - mysql-libs=9.0.1 - ncurses=6.5 - - netcdf4=1.6.0 + - netcdf4=1.7.1 - nodeenv=1.9.1 - - numpy=1.26.4 - - olefile=0.47 - - openjpeg=2.5.0 + - numpy=2.1.1 + - openjpeg=2.5.2 + - openldap=2.6.8 - openssl=3.3.2 - - orc=2.0.1 + - orc=2.0.2 - packaging=24.1 - pandas=2.2.3 - parso=0.8.4 - partd=1.4.2 - pathlib=1.0.1 - pathspec=0.12.1 - - pcre2=10.43 + - pcre2=10.44 - pdbufr=0.11.0 - pexpect=4.9.0 - pickleshare=0.7.5 - - pillow=8.4.0 + - pillow=10.4.0 - pip=24.2 + - pixman=0.43.2 + - pkgutil-resolve-name=1.3.10 - platformdirs=4.3.6 - pluggy=1.5.0 - pre-commit=3.8.0 @@ -202,8 +233,8 @@ dependencies: - pthread-stubs=0.4 - ptyprocess=0.7.0 - pure_eval=0.2.3 - - pyarrow=16.1.0 - - pyarrow-core=16.1.0 + - pyarrow=17.0.0 + - pyarrow-core=17.0.0 - pyarrow-hotfix=0.6 - pycodestyle=2.12.1 - pycparser=2.22 @@ -214,40 +245,39 @@ dependencies: - pygments=2.18.0 - pylint=3.3.1 - pyparsing=3.1.4 - - pyqt=5.15.4 - - pyqt5-sip=12.9.0 + - pyside6=6.7.3 - pysocks=1.7.1 - pytest=8.3.3 - - python=3.10.14 + - python=3.12.6 - python-dateutil=2.9.0 - - python-eccodes=1.4.0 + - python-eccodes=1.7.1 - python-tzdata=2024.2 - - python_abi=3.10 + - python_abi=3.12 - pytoolconfig=1.2.5 - pytz=2024.1 - pyyaml=6.0.2 - qhull=2020.2 - - qt-main=5.15.2 + - qt6-main=6.7.3 - re2=2023.09.01 - readline=8.2 + - referencing=0.35.1 - regex=2024.9.11 - requests=2.32.3 - rich=13.9.1 - rope=1.13.0 + - rpds-py=0.20.0 - rstcheck=6.2.4 - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 - ruamel.yaml.clib=0.2.8 - - s2n=1.4.15 + - s2n=1.5.3 - scipy=1.14.1 - setuptools=75.1.0 - shellingham=1.5.4 - - sip=6.5.1 - six=1.16.0 - snappy=1.2.1 - snowballstemmer=2.2.0 - sortedcontainers=2.4.0 - - sqlite=3.46.0 - stack_data=0.6.2 - tblib=3.0.0 - tk=8.6.13 @@ -265,19 +295,34 @@ dependencies: - typing_extensions=4.12.2 - tzdata=2024a - ukkonen=1.0.1 - - unicodedata2=15.1.0 - urllib3=2.2.3 - virtualenv=20.26.6 + - wayland=1.23.1 - wcwidth=0.2.13 - wheel=0.44.0 - xarray=2024.9.0 + - xcb-util=0.4.1 + - xcb-util-cursor=0.1.5 + - xcb-util-image=0.4.0 + - xcb-util-keysyms=0.4.1 + - xcb-util-renderutil=0.3.10 + - xcb-util-wm=0.4.2 - xkeyboard-config=2.43 + - xorg-libice=1.1.1 + - xorg-libsm=1.2.4 - xorg-libx11=1.8.10 - xorg-libxau=1.0.11 + - xorg-libxcomposite=0.4.6 + - xorg-libxcursor=1.2.2 + - xorg-libxdamage=1.1.6 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 - xorg-libxfixes=6.0.1 - xorg-libxi=1.8.2 + - xorg-libxrandr=1.5.4 + - xorg-libxrender=0.9.11 + - xorg-libxtst=1.2.5 + - xorg-libxxf86vm=1.1.5 - xorg-xextproto=7.3.0 - xorg-xorgproto=2024.1 - xyzservices=2024.9.0 @@ -285,7 +330,7 @@ dependencies: - yaml=0.2.5 - zict=3.0.0 - zipp=3.20.2 - - zlib=1.2.13 + - zlib=1.3.1 - zstandard=0.23.0 - zstd=1.5.6 - pip: diff --git a/requirements/requirements.yml b/requirements/requirements.yml index 09ae938c..bdccfb63 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -7,7 +7,7 @@ dependencies: # runtime - click>=7.1.2 - earthkit-data - - eccodes=2.25.0 + - eccodes=2.35.0 - Jinja2>=3.0.1 - matplotlib>=3.2.1 - netCDF4>=1.5.3 diff --git a/setup_env.sh b/setup_env.sh index 898e3579..46b3968c 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -74,7 +74,7 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEFINITION_VERSION="v2.25.0.2" +DEFINITION_VERSION="v2.35.0.1dm1" DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} From 59a08339bc65f362ead8c85d2434feb991db02fa Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Mon, 7 Oct 2024 15:08:29 +0200 Subject: [PATCH 46/62] fix python version --- .github/workflows/pre-commit.yml | 2 +- .github/workflows/pytest.yml | 2 +- requirements/environment.yml | 33 ++++++++++++++++---------------- requirements/requirements.yml | 4 ++-- setup_env.sh | 4 ++-- 5 files changed, 23 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index d95d709c..743e67e6 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.12.6 + python-version: 3.10.8 - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 2fc2caa4..5967d4aa 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v2 with: - python-version: 3.12.6 + python-version: 3.10.8 - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" diff --git a/requirements/environment.yml b/requirements/environment.yml index a796f19c..9319c692 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -7,7 +7,7 @@ dependencies: - _openmp_mutex=4.5 - alsa-lib=1.2.12 - annotated-types=0.7.0 - - astroid=3.3.4 + - astroid=3.3.5 - asttokens=2.4.1 - attrs=24.2.0 - aws-c-auth=0.7.31 @@ -51,7 +51,7 @@ dependencies: - contourpy=1.3.0 - cycler=0.12.1 - cyrus-sasl=2.1.27 - - cytoolz=0.12.3 + - cytoolz=1.0.0 - dask=2024.9.1 - dask-core=2024.9.1 - dask-expr=1.1.15 @@ -62,9 +62,9 @@ dependencies: - distributed=2024.9.1 - docutils=0.21.2 - double-conversion=3.3.0 - - earthkit-data=0.10.4 + - earthkit-data=0.10.6 - earthkit-geo=0.2.0 - - eccodes=2.35.0 + - eccodes=2.38.0 - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 @@ -90,7 +90,7 @@ dependencies: - h2=4.1.0 - harfbuzz=9.0.0 - hdf4=4.2.15 - - hdf5=1.14.3 + - hdf5=1.14.4 - hpack=4.0.0 - hyperframe=6.0.1 - icu=75.1 @@ -130,7 +130,7 @@ dependencies: - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.10.1 - - libdeflate=1.21 + - libdeflate=1.22 - libdrm=2.4.123 - libedit=3.1.20191231 - libegl=1.7.0 @@ -205,7 +205,7 @@ dependencies: - ncurses=6.5 - netcdf4=1.7.1 - nodeenv=1.9.1 - - numpy=2.1.1 + - numpy=2.1.2 - openjpeg=2.5.2 - openldap=2.6.8 - openssl=3.3.2 @@ -226,8 +226,8 @@ dependencies: - pkgutil-resolve-name=1.3.10 - platformdirs=4.3.6 - pluggy=1.5.0 - - pre-commit=3.8.0 - - pre-commit-hooks=4.6.0 + - pre-commit=4.0.0 + - pre-commit-hooks=5.0.0 - prompt-toolkit=3.0.48 - psutil=6.0.0 - pthread-stubs=0.4 @@ -248,11 +248,11 @@ dependencies: - pyside6=6.7.3 - pysocks=1.7.1 - pytest=8.3.3 - - python=3.12.6 + - python=3.10.8 - python-dateutil=2.9.0 - - python-eccodes=1.7.1 + - python-eccodes=2.37.0 - python-tzdata=2024.2 - - python_abi=3.12 + - python_abi=3.10 - pytoolconfig=1.2.5 - pytz=2024.1 - pyyaml=6.0.2 @@ -263,14 +263,14 @@ dependencies: - referencing=0.35.1 - regex=2024.9.11 - requests=2.32.3 - - rich=13.9.1 + - rich=13.9.2 - rope=1.13.0 - rpds-py=0.20.0 - rstcheck=6.2.4 - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 - ruamel.yaml.clib=0.2.8 - - s2n=1.5.3 + - s2n=1.5.4 - scipy=1.14.1 - setuptools=75.1.0 - shellingham=1.5.4 @@ -284,7 +284,7 @@ dependencies: - toml=0.10.2 - tomli=2.0.2 - tomlkit=0.13.2 - - toolz=0.12.1 + - toolz=1.0.0 - tornado=6.4.1 - tqdm=4.66.5 - traitlets=5.14.3 @@ -293,8 +293,9 @@ dependencies: - typer-slim-standard=0.12.5 - typing-extensions=4.12.2 - typing_extensions=4.12.2 - - tzdata=2024a + - tzdata=2024b - ukkonen=1.0.1 + - unicodedata2=15.1.0 - urllib3=2.2.3 - virtualenv=20.26.6 - wayland=1.23.1 diff --git a/requirements/requirements.yml b/requirements/requirements.yml index bdccfb63..0764cc8e 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -2,12 +2,12 @@ channels: - conda-forge - defaults dependencies: - - python>=3.10 + - python==3.10.8 - pip>=22.3 # runtime - click>=7.1.2 - earthkit-data - - eccodes=2.35.0 + - eccodes - Jinja2>=3.0.1 - matplotlib>=3.2.1 - netCDF4>=1.5.3 diff --git a/setup_env.sh b/setup_env.sh index 46b3968c..3ccf61be 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -13,7 +13,7 @@ DEFAULT_ENV_NAME="probtest" # Default options ENV_NAME="${DEFAULT_ENV_NAME}" -PYVERSION=3.12 +PYVERSION=3.10 PINNED=true EXPORT=false CONDA=conda @@ -74,7 +74,7 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEFINITION_VERSION="v2.35.0.1dm1" +DEFINITION_VERSION="v2.25.0.2" DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} From 21c6d24014e17f5a7e5d8f8cfe3959e8a60b8474 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Mon, 7 Oct 2024 15:57:25 +0200 Subject: [PATCH 47/62] might not be needed --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ad18150a..721ba838 100644 --- a/README.md +++ b/README.md @@ -107,7 +107,6 @@ setup scripts: ```console ./setup_miniconda.sh -source miniconda/bin/activate ./setup_env.sh -n probtest -u ``` From 8000bdf0881d8f729f766e9bdb65205aff1784c0 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Tue, 8 Oct 2024 17:19:08 +0200 Subject: [PATCH 48/62] revert hotfix --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5d12020f..ae63ad84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -121,7 +121,7 @@ repos: types: [python] args: - "--max-line-length=88" - - "--disable=C0116,R0912,R0913,R0914,R0915,R0917,R1710,W0511,W0719" + - "--disable=C0116,R0912,R0913,R0914,R0915,R1710,W0511,W0719" - repo: local hooks: - id: flake8 From 43a39857ba8c2a869edcf55d11fdaf38991bb217 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 16 Oct 2024 10:22:02 +0200 Subject: [PATCH 49/62] update README.md --- README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 7b3a07f9..f0bf08bd 100644 --- a/README.md +++ b/README.md @@ -100,10 +100,13 @@ The pinned requirements can be installed by ```console ./setup_env.sh ``` +which is recommended for users. + The unpinned requirements and updating the environment can be done by ```console ./setup_env.sh -u -e ``` +which is recommended for developers and required for adding new requirements. ### The init command @@ -138,7 +141,17 @@ here should refer to your experiment script: ```console cd icon-base-dir/reference-build -python ../externals/probtest/probtest.py init --codebase-install $PWD --experiment-name exp_name --reference $PWD --file-id NetCDF "*atm_3d_ml*.nc" --file-id NetCDF "*atm_3d_il*.nc" --file-id NetCDF "*atm_3d_hl*.nc" --file-id NetCDF "*atm_3d_pl*.nc" --file-id latlon "*atm_2d_ll*.nc" --file-id meteogram "Meteogram*.nc" +python ../externals/probtest/probtest.py init \ + --codebase-install $PWD \ + --experiment-name exp_name \ + --reference $PWD \ + --file-id NetCDF "*atm_3d_ml*.nc" \ + --file-id NetCDF "*atm_3d_il*.nc" \ + --file-id NetCDF "*atm_3d_hl*.nc" \ + --file-id NetCDF "*atm_3d_pl*.nc" \ + --file-id latlon "*atm_2d_ll*.nc" \ + --file-id meteogram "Meteogram*.nc" \ + --file-id GRIB "lfff0*z" ``` You might need to update the used account in the json file. The perturbation amplitude may also need to be changed in the json file From 2ed28453fe41c1146a79a44d920553a88ee649c6 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 16 Oct 2024 15:28:40 +0200 Subject: [PATCH 50/62] refactor --- engine/cdo_table.py | 6 +-- tests/util/test_model_output_parser.py | 8 ++-- util/model_output_parser.py | 54 ++++++++++++++++++++------ 3 files changed, 50 insertions(+), 18 deletions(-) diff --git a/engine/cdo_table.py b/engine/cdo_table.py index 95d0d113..6040efc4 100644 --- a/engine/cdo_table.py +++ b/engine/cdo_table.py @@ -139,9 +139,9 @@ def cdo_table( assert isinstance(file_specification, dict), "must be dict" # save original method and restore at the end of this module - dataframe_from_ncfile_orig = model_output_parser.dataframe_from_ncfile + dataframe_from_ncfile_orig = model_output_parser.create_statistics_dataframe # modify netcdf parse method: - model_output_parser.dataframe_from_ncfile = rel_diff_stats + model_output_parser.create_statistics_dataframe = rel_diff_stats # step 1: compute rel-diff netcdf files with tempfile.TemporaryDirectory() as tmpdir: @@ -202,4 +202,4 @@ def cdo_table( Path(cdo_table_file).parent.mkdir(parents=True, exist_ok=True) df.to_csv(cdo_table_file) - model_output_parser.dataframe_from_ncfile = dataframe_from_ncfile_orig + model_output_parser.create_statistics_dataframe = dataframe_from_ncfile_orig diff --git a/tests/util/test_model_output_parser.py b/tests/util/test_model_output_parser.py index e54fee20..fbda4b65 100644 --- a/tests/util/test_model_output_parser.py +++ b/tests/util/test_model_output_parser.py @@ -6,7 +6,7 @@ import pytest -from util.model_output_parser import get_ds +from util.model_output_parser import get_dataset @pytest.fixture(name="mock_ds_grib") @@ -39,7 +39,7 @@ def test_get_ds_success(mock_ds_grib): pid = 1 lev = "surface" - result = get_ds(mock_ds_grib, pid, lev) + result = get_dataset(mock_ds_grib, pid, lev) # Ensure the dataset is selected once mock_ds_grib.sel.assert_called_once_with(paramId=pid, typeOfLevel=lev) @@ -88,7 +88,7 @@ def test_get_ds_recursive_selection( mock_ds_grib.sel.return_value.to_xarray.side_effect = to_xarray_return_value - result = get_ds(mock_ds_grib, pid, lev) + result = get_dataset(mock_ds_grib, pid, lev) # Ensure the recursive logic is triggered by calling sel multiple times assert mock_ds_grib.sel.call_count >= len(to_xarray_return_value) @@ -107,7 +107,7 @@ def test_get_ds_keyerror_handling(caplog, mock_ds_grib): # Simulate KeyErrors for all attempts to select datasets mock_ds_grib.sel.return_value.to_xarray.side_effect = KeyError() - result = get_ds(mock_ds_grib, pid, lev) + result = get_dataset(mock_ds_grib, pid, lev) # Assert that the warning was logged assert "GRIB file of level surface and paramId 1 cannot be read." in caplog.text diff --git a/util/model_output_parser.py b/util/model_output_parser.py index ed0fdf6c..01a92993 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -49,7 +49,7 @@ def parse_netcdf(file_id, filename, specification): var_dfs = [] for v in var_tmp: - sub_df = dataframe_from_ncfile( + sub_df = create_statistics_dataframe( file_id=file_id, filename=filename, varname=v, @@ -73,20 +73,20 @@ def parse_grib(file_id, filename, specification): ds_grib = earthkit.data.from_source("file", filename) short_name_excl = specification["var_excl"] - short_name = np.unique(ds_grib.metadata("shortName")) - short_name = short_name[ - np.isin(short_name, short_name_excl, invert=True, assume_unique=True) + short_names = np.unique(ds_grib.metadata("shortName")) + short_names = short_names[ + np.isin(short_names, short_name_excl, invert=True, assume_unique=True) ].tolist() - level_type = np.unique(ds_grib.metadata("typeOfLevel")).tolist() + level_types = np.unique(ds_grib.metadata("typeOfLevel")).tolist() var_dfs = [] - for lev in level_type: + for lev in level_types: param_id = np.unique( - ds_grib.sel(typeOfLevel=lev, shortName=short_name).metadata("paramId") + ds_grib.sel(typeOfLevel=lev, shortName=short_names).metadata("paramId") ).tolist() for pid in param_id: - ds_temp_list = get_ds(ds_grib, pid, lev) + ds_temp_list = get_dataset(ds_grib, pid, lev) for ds_temp in ds_temp_list: v = list(ds_temp.keys())[0] @@ -97,7 +97,7 @@ def parse_grib(file_id, filename, specification): ] ds = ds_temp.squeeze(dim=dim_to_squeeze) - sub_df = dataframe_from_ncfile( + sub_df = create_statistics_dataframe( file_id=file_id, filename=filename, varname=v, @@ -111,7 +111,7 @@ def parse_grib(file_id, filename, specification): return var_dfs -def get_ds(ds_grib, pid, lev): +def get_dataset(ds_grib, pid, lev): """ Retrieve datasets from a GRIB file based on specified parameters and hierarchical metadata. @@ -208,9 +208,41 @@ def __get_variables(data, time_dim, horizontal_dims): return variables -def dataframe_from_ncfile( +def create_statistics_dataframe( file_id, filename, varname, time_dim, horizontal_dims, xarray_ds, fill_value_key ): # pylint: disable=too-many-positional-arguments + """ + Create a DataFrame of statistical values for a given variable from an xarray + dataset. + + This function computes statistics (mean, max, min, etc.) over horizontal + dimensions and organizes them into a pandas DataFrame, indexed by file ID, + variable name, and height (if applicable). + The columns represent time and the computed statistics. + + Parameters: + ----------- + file_id : str + Identifier for the file. + filename : str + Name of the input file. + varname : str + Name of the variable to process. + time_dim : str + Name of the time dimension. + horizontal_dims : list + List of dimensions to compute statistics over. + xarray_ds : xarray.Dataset + The xarray dataset containing the data. + fill_value_key : str + Key for the fill value in the dataset. + + Returns: + -------- + pd.DataFrame + DataFrame with the computed statistics indexed by file ID, variable, and + height. + """ statistics = statistics_over_horizontal_dim( xarray_ds[varname], horizontal_dims, From 586d5a10b76e469753a53d71a2ce24196d71e5a0 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 16 Oct 2024 16:17:56 +0200 Subject: [PATCH 51/62] use closets possiblle eccodes to 2.25.0 --- requirements/environment.yml | 79 ++++++++++++++++------------------- requirements/requirements.yml | 2 +- setup_env.sh | 2 +- 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 72ca18e1..d3d5283a 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -23,19 +23,19 @@ dependencies: - aws-checksums=0.1.20 - aws-crt-cpp=0.28.3 - aws-sdk-cpp=1.11.407 - - azure-core-cpp=1.13.0 - - azure-identity-cpp=1.9.0 + - azure-core-cpp=1.14.0 + - azure-identity-cpp=1.10.0 - azure-storage-blobs-cpp=12.13.0 - azure-storage-common-cpp=12.8.0 - azure-storage-files-datalake-cpp=12.12.0 - - black=24.8.0 + - black=24.10.0 - blosc=1.21.6 - - bokeh=3.5.2 + - bokeh=3.6.0 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 - bzip2=1.0.8 - - c-ares=1.33.1 + - c-ares=1.34.2 - ca-certificates=2024.8.30 - cairo=1.18.0 - certifi=2024.8.30 @@ -43,9 +43,9 @@ dependencies: - cfgrib=0.9.14.1 - cfgv=3.3.1 - cftime=1.6.4 - - charset-normalizer=3.3.2 + - charset-normalizer=3.4.0 - click=8.1.7 - - cloudpickle=3.0.0 + - cloudpickle=3.1.0 - codespell=2.3.0 - colorama=0.4.6 - contourpy=1.3.0 @@ -58,13 +58,12 @@ dependencies: - dbus=1.13.6 - decorator=5.1.1 - dill=0.3.9 - - distlib=0.3.8 + - distlib=0.3.9 - distributed=2024.9.1 - docutils=0.21.2 - double-conversion=3.3.0 - - earthkit-data=0.10.6 - - earthkit-geo=0.2.0 - - eccodes=2.38.0 + - earthkit-data=0.5.6 + - eccodes=2.33.0 - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 @@ -90,7 +89,7 @@ dependencies: - h2=4.1.0 - harfbuzz=9.0.0 - hdf4=4.2.15 - - hdf5=1.14.4 + - hdf5=1.14.3 - hpack=4.0.0 - hyperframe=6.0.1 - icu=75.1 @@ -98,7 +97,6 @@ dependencies: - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 - - importlib_resources=6.4.5 - iniconfig=2.0.0 - ipdb=0.13.13 - ipython=8.28.0 @@ -106,8 +104,6 @@ dependencies: - jasper=4.2.4 - jedi=0.19.1 - jinja2=3.1.4 - - jsonschema=4.23.0 - - jsonschema-specifications=2023.12.1 - keyutils=1.6.1 - kiwisolver=1.4.7 - krb5=1.21.3 @@ -125,8 +121,8 @@ dependencies: - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 - - libclang-cpp19.1=19.1.0 - - libclang13=19.1.0 + - libclang-cpp19.1=19.1.1 + - libclang13=19.1.1 - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.10.1 @@ -138,24 +134,24 @@ dependencies: - libevent=2.1.12 - libexpat=2.6.3 - libffi=3.4.2 - - libgcc=14.1.0 - - libgcc-ng=14.1.0 - - libgfortran=14.1.0 - - libgfortran-ng=14.1.0 - - libgfortran5=14.1.0 + - libgcc=14.2.0 + - libgcc-ng=14.2.0 + - libgfortran=14.2.0 + - libgfortran-ng=14.2.0 + - libgfortran5=14.2.0 - libgl=1.7.0 - libglib=2.82.1 - libglu=9.0.0 - libglvnd=1.7.0 - libglx=1.7.0 - - libgomp=14.1.0 - - libgoogle-cloud=2.29.0 - - libgoogle-cloud-storage=2.29.0 + - libgomp=14.2.0 + - libgoogle-cloud=2.30.0 + - libgoogle-cloud-storage=2.30.0 - libgrpc=1.65.5 - libiconv=1.17 - libjpeg-turbo=3.0.0 - liblapack=3.9.0 - - libllvm19=19.1.1 + - libllvm19=19.1.2 - libnetcdf=4.9.2 - libnghttp2=1.58.0 - libnsl=2.0.1 @@ -167,11 +163,11 @@ dependencies: - libpng=1.6.44 - libpq=17.0 - libprotobuf=5.27.5 - - libre2-11=2023.09.01 + - libre2-11=2024.07.02 - libsqlite=3.46.1 - libssh2=1.11.0 - - libstdcxx=14.1.0 - - libstdcxx-ng=14.1.0 + - libstdcxx=14.2.0 + - libstdcxx-ng=14.2.0 - libthrift=0.21.0 - libtiff=4.7.0 - libutf8proc=2.8.0 @@ -184,12 +180,10 @@ dependencies: - libzip=1.11.1 - libzlib=1.3.1 - locket=1.0.0 - - lru-dict=1.3.0 - lz4=4.3.3 - lz4-c=1.9.4 - - markdown=3.6 - markdown-it-py=3.0.0 - - markupsafe=3.0.0 + - markupsafe=3.0.1 - matplotlib=3.9.2 - matplotlib-base=3.9.2 - matplotlib-inline=0.1.7 @@ -197,7 +191,7 @@ dependencies: - mdurl=0.1.2 - msgpack-python=1.1.0 - munkres=1.1.4 - - mypy=1.11.2 + - mypy=1.12.0 - mypy_extensions=1.0.0 - mysql-common=9.0.1 - mysql-libs=9.0.1 @@ -222,10 +216,9 @@ dependencies: - pillow=10.4.0 - pip=24.2 - pixman=0.43.2 - - pkgutil-resolve-name=1.3.10 - platformdirs=4.3.6 - pluggy=1.5.0 - - pre-commit=4.0.0 + - pre-commit=4.0.1 - pre-commit-hooks=5.0.0 - prompt-toolkit=3.0.48 - psutil=6.0.0 @@ -243,33 +236,31 @@ dependencies: - pyflakes=3.2.0 - pygments=2.18.0 - pylint=3.3.1 - - pyparsing=3.1.4 - - pyside6=6.7.3 + - pyparsing=3.2.0 + - pyside6=6.8.0 - pysocks=1.7.1 - pytest=8.3.3 - python=3.10.8 - python-dateutil=2.9.0 - - python-eccodes=2.37.0 + - python-eccodes=1.7.1 - python-tzdata=2024.2 - python_abi=3.10 - pytoolconfig=1.2.5 - pytz=2024.1 - pyyaml=6.0.2 - qhull=2020.2 - - qt6-main=6.7.3 - - re2=2023.09.01 + - qt6-main=6.8.0 + - re2=2024.07.02 - readline=8.2 - - referencing=0.35.1 - regex=2024.9.11 - requests=2.32.3 - rich=13.9.2 - rope=1.13.0 - - rpds-py=0.20.0 - rstcheck=6.2.4 - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 - ruamel.yaml.clib=0.2.8 - - s2n=1.5.4 + - s2n=1.5.5 - scipy=1.14.1 - setuptools=75.1.0 - shellingham=1.5.4 @@ -334,6 +325,6 @@ dependencies: - zstandard=0.23.0 - zstd=1.5.6 - pip: - - coverage==7.6.1 + - coverage==7.6.3 - flake8-pyproject==1.2.3 - pytest-cov==5.0.0 diff --git a/requirements/requirements.yml b/requirements/requirements.yml index 0764cc8e..49a8ea58 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -7,7 +7,7 @@ dependencies: # runtime - click>=7.1.2 - earthkit-data - - eccodes + - eccodes==2.33.0 - Jinja2>=3.0.1 - matplotlib>=3.2.1 - netCDF4>=1.5.3 diff --git a/setup_env.sh b/setup_env.sh index c6dfd9ac..45a0fe36 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -74,7 +74,7 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEFINITION_VERSION="v2.25.0.2" +DEFINITION_VERSION="v2.33.0.5d" DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} From 9b94a3ab17c59f818e37a5b787321cc628b21e54 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 16 Oct 2024 16:33:17 +0200 Subject: [PATCH 52/62] refactor --- engine/cdo_table.py | 6 +++--- engine/performance.py | 6 ++++-- util/dataframe_ops.py | 16 +++++++++------- util/file_system.py | 3 ++- util/model_output_parser.py | 4 ++-- 5 files changed, 20 insertions(+), 15 deletions(-) diff --git a/engine/cdo_table.py b/engine/cdo_table.py index 6040efc4..70ee82ba 100644 --- a/engine/cdo_table.py +++ b/engine/cdo_table.py @@ -17,7 +17,7 @@ from util.click_util import CommaSeperatedInts, cli_help from util.constants import cdo_bins from util.dataframe_ops import df_from_file_ids -from util.file_system import file_names_from_pattern +from util.file_system import get_file_names_from_pattern from util.log_handler import logger @@ -146,14 +146,14 @@ def cdo_table( # step 1: compute rel-diff netcdf files with tempfile.TemporaryDirectory() as tmpdir: for _, file_pattern in file_id: - ref_files, err = file_names_from_pattern(model_output_dir, file_pattern) + ref_files, err = get_file_names_from_pattern(model_output_dir, file_pattern) if err > 0: logger.info( "did not find any files for pattern %s. Continue.", file_pattern ) continue ref_files.sort() - perturb_files, err = file_names_from_pattern( + perturb_files, err = get_file_names_from_pattern( perturbed_model_output_dir.format(member_id=member_id), file_pattern ) if err > 0: diff --git a/engine/performance.py b/engine/performance.py index 6edf2bce..82814e9d 100644 --- a/engine/performance.py +++ b/engine/performance.py @@ -15,7 +15,7 @@ import click from util.click_util import cli_help -from util.file_system import file_names_from_pattern +from util.file_system import get_file_names_from_pattern from util.icon.extract_timings import read_logfile from util.log_handler import logger from util.tree import TREEFILE_TEMPLATE, TimingTree @@ -36,7 +36,9 @@ def performance(timing_regex, timing_database, append_time): if timing_file_name_base == "": timing_file_name_base = "." - timing_file_name, err = file_names_from_pattern(timing_file_name_base, timing_regex) + timing_file_name, err = get_file_names_from_pattern( + timing_file_name_base, timing_regex + ) if err > 0: logger.info("Did not find any files for regex %s", timing_regex) sys.exit(1) diff --git a/util/dataframe_ops.py b/util/dataframe_ops.py index 711ac0e0..c66ffeca 100644 --- a/util/dataframe_ops.py +++ b/util/dataframe_ops.py @@ -12,7 +12,7 @@ import pandas as pd from util.constants import CHECK_THRESHOLD, compute_statistics -from util.file_system import file_names_from_pattern +from util.file_system import get_file_names_from_pattern from util.log_handler import logger from util.model_output_parser import model_output_parser @@ -88,6 +88,13 @@ def read_input_file(label, file_name, specification): def df_from_file_ids(file_id, input_dir, file_specification): """ + Collect data frames for each combination of file id (fid) and specification + (spec). + Frames for the same fid and spec represent different timestamps and have to + be concatenated along time-axis (axis=1). + Time-concatenated frames from different ids and/or specifications will be + concatenated along variable-axis (axis=0). + file_id: [[file_type, file_pattern], [file_type, file_pattern], ...] List of 2-tuples. The 2-tuple combines two strings. The first sets the file_type and must be a key in file_specification. The second string @@ -111,14 +118,9 @@ def df_from_file_ids(file_id, input_dir, file_specification): separated by ":". """ - # Collect data frames for each combination of file id (fid) and - # specification (spec). Frames for the same fid and spec represent - # different timestamps and have to be concatenated along time-axis (axis=1). - # Time-concatenated frames from different ids and/or specifications will be - # concatenated along variable-axis (axis=0). fid_dfs = [] for file_type, file_pattern in file_id: - input_files, err = file_names_from_pattern(input_dir, file_pattern) + input_files, err = get_file_names_from_pattern(input_dir, file_pattern) if err > 0: logger.info( "Can not find any files for file_pattern %s. Continue.", file_pattern diff --git a/util/file_system.py b/util/file_system.py index 99ad3baf..8dda9e73 100644 --- a/util/file_system.py +++ b/util/file_system.py @@ -7,7 +7,7 @@ from util.log_handler import logger -def file_names_from_pattern(dir_name, file_pattern): +def get_file_names_from_pattern(dir_name, file_pattern): """ Search for all file patching file_pattern in directory dir_name @@ -38,5 +38,6 @@ def file_names_from_pattern(dir_name, file_pattern): for f in Path(dir_name).glob("*"): logger.debug(f.name) err = 1 + file_names = sorted(file_names) return file_names, err diff --git a/util/model_output_parser.py b/util/model_output_parser.py index 01a92993..4a2d8ea4 100644 --- a/util/model_output_parser.py +++ b/util/model_output_parser.py @@ -82,10 +82,10 @@ def parse_grib(file_id, filename, specification): var_dfs = [] for lev in level_types: - param_id = np.unique( + param_ids = np.unique( ds_grib.sel(typeOfLevel=lev, shortName=short_names).metadata("paramId") ).tolist() - for pid in param_id: + for pid in param_ids: ds_temp_list = get_dataset(ds_grib, pid, lev) for ds_temp in ds_temp_list: v = list(ds_temp.keys())[0] From f51fcd40036ea3b0a731e9cb943f097971267917 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Wed, 16 Oct 2024 17:27:07 +0200 Subject: [PATCH 53/62] not so nice workaround --- util/dataframe_ops.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/util/dataframe_ops.py b/util/dataframe_ops.py index c66ffeca..1f0e4a04 100644 --- a/util/dataframe_ops.py +++ b/util/dataframe_ops.py @@ -178,7 +178,21 @@ def unify_time_index(fid_dfs): unique_times = list(df.columns.levels[time_multiindex_index]) unique_times.sort() - df = df.reindex(columns=unique_times, level=time_multiindex_index) + # workaround needed unitl grib time is read in properly + try: + df = df.reindex(columns=unique_times, level=time_multiindex_index) + except ValueError: + logger.warning( + ( + "cannot reindex on an axis with duplicate labels:" + "unique_times (%s), " + "time_multiindex_index (%s), " + "df.columns (%s)." + ), + unique_times, + time_multiindex_index, + df.columns, + ) df.columns = df.columns.set_levels(range(ntime), level="time") From e50ec27fc373323c375a29ac46fa71539f5bdcdb Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 17 Oct 2024 16:02:10 +0200 Subject: [PATCH 54/62] workaround --- tests/util/test_dataframe_ops.py | 33 +++++++++++++- util/dataframe_ops.py | 75 +++++++++++++++++++++++++------- 2 files changed, 91 insertions(+), 17 deletions(-) diff --git a/tests/util/test_dataframe_ops.py b/tests/util/test_dataframe_ops.py index ec58928c..306e2b24 100644 --- a/tests/util/test_dataframe_ops.py +++ b/tests/util/test_dataframe_ops.py @@ -4,9 +4,40 @@ from unittest.mock import patch +import numpy as np import pandas as pd -from util.dataframe_ops import parse_check +from util.dataframe_ops import adjust_time_index, parse_check + + +def test_adjust_time_index(): + # Create a sample DataFrame with MultiIndex for 'time' and 'statistic' + index = pd.MultiIndex.from_product( + [["0 days 00:00:00", "0 days 00:01:00"], ["mean", "max", "min"]], + names=["time", "statistic"], + ) + data = np.random.random((5, 6)) # Sample data + df = pd.DataFrame(data, columns=index) + + # Apply the function to a list of DataFrames (in this case, just one DataFrame) + result = adjust_time_index([df])[0] + + # Create the expected new time index based on unique statistics + expected_time_values = np.repeat( + range(2), 3 + ) # Two unique time points, three statistics per time + expected_index = pd.MultiIndex.from_arrays( + [expected_time_values, ["mean", "max", "min"] * 2], + names=["time", "statistic"], + ) + + # Verify that the new MultiIndex matches the expected index + assert result.columns.equals( + expected_index + ), "The time index was not adjusted correctly." + + # Verify that the data remains unchanged + pd.testing.assert_frame_equal(df, result, check_like=True) @patch("util.dataframe_ops.parse_probtest_csv") diff --git a/util/dataframe_ops.py b/util/dataframe_ops.py index 1f0e4a04..3a573f0f 100644 --- a/util/dataframe_ops.py +++ b/util/dataframe_ops.py @@ -154,7 +154,12 @@ def df_from_file_ids(file_id, input_dir, file_specification): logger.error("Could not find any file.") sys.exit(2) - fid_dfs = unify_time_index(fid_dfs) + # workaround for not properly set time column + try: + fid_dfs = unify_time_index(fid_dfs) + except ValueError: + fid_dfs = adjust_time_index(fid_dfs) + # different file IDs will have different variables but with same timestamps: # concatenate along variable axis df = pd.concat(fid_dfs, axis=0) @@ -178,21 +183,7 @@ def unify_time_index(fid_dfs): unique_times = list(df.columns.levels[time_multiindex_index]) unique_times.sort() - # workaround needed unitl grib time is read in properly - try: - df = df.reindex(columns=unique_times, level=time_multiindex_index) - except ValueError: - logger.warning( - ( - "cannot reindex on an axis with duplicate labels:" - "unique_times (%s), " - "time_multiindex_index (%s), " - "df.columns (%s)." - ), - unique_times, - time_multiindex_index, - df.columns, - ) + df = df.reindex(columns=unique_times, level=time_multiindex_index) df.columns = df.columns.set_levels(range(ntime), level="time") @@ -201,6 +192,58 @@ def unify_time_index(fid_dfs): return fid_dfs_out +def adjust_time_index(fid_dfs): + """ + Adjust the 'time' level of the MultiIndex in DataFrame columns by replacing + it with a sequential range based on the number of unique 'statistic' values. + + Parameters: + ----------- + fid_dfs : list of pandas.DataFrame + A list of DataFrames with MultiIndex columns containing 'time' and + 'statistic' levels. + + Returns: + -------- + fid_dfs_out : list of pandas.DataFrame + DataFrames with corrected 'time' values in their MultiIndex columns. + """ + fid_dfs_out = [] + for df in fid_dfs: + # Get the existing MultiIndex + current_multiindex = df.columns + + # Find the number of unique values in the 'statistic' level + unique_statistic_count = current_multiindex.get_level_values( + "statistic" + ).nunique() + + # Create a sequential integer range for 'time' values (based on the + # number of unique statistics) + new_time_values = list( + range( + len(current_multiindex.get_level_values("time")) + // unique_statistic_count + ) + ) + + # Repeat these integer values to match the length of your columns + new_time_repeated = np.repeat(new_time_values, unique_statistic_count) + + # Construct a new MultiIndex with updated 'time' values + new_multiindex = pd.MultiIndex.from_arrays( + [new_time_repeated, current_multiindex.get_level_values("statistic")], + names=["time", "statistic"], + ) + + # Assign the new MultiIndex to the DataFrame + df.columns = new_multiindex + + fid_dfs_out.append(df) + + return fid_dfs_out + + def check_intersection(df_ref, df_cur): # Check if variable names in reference and test case have any intersection # Check if numbers of time steps agree From 427a9547ba4e43c7e387f00584426a0099dacdb9 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 7 Nov 2024 15:03:48 +0100 Subject: [PATCH 55/62] update env --- requirements/environment.yml | 117 +++++++++++++++++------------------ 1 file changed, 58 insertions(+), 59 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index d3d5283a..d9e72f30 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -10,18 +10,18 @@ dependencies: - astroid=3.3.5 - asttokens=2.4.1 - attrs=24.2.0 - - aws-c-auth=0.7.31 - - aws-c-cal=0.7.4 - - aws-c-common=0.9.28 - - aws-c-compression=0.2.19 - - aws-c-event-stream=0.4.3 - - aws-c-http=0.8.10 - - aws-c-io=0.14.18 - - aws-c-mqtt=0.10.7 - - aws-c-s3=0.6.6 - - aws-c-sdkutils=0.1.19 - - aws-checksums=0.1.20 - - aws-crt-cpp=0.28.3 + - aws-c-auth=0.8.0 + - aws-c-cal=0.8.0 + - aws-c-common=0.10.0 + - aws-c-compression=0.3.0 + - aws-c-event-stream=0.5.0 + - aws-c-http=0.9.0 + - aws-c-io=0.15.0 + - aws-c-mqtt=0.11.0 + - aws-c-s3=0.7.0 + - aws-c-sdkutils=0.2.0 + - aws-checksums=0.2.0 + - aws-crt-cpp=0.29.1 - aws-sdk-cpp=1.11.407 - azure-core-cpp=1.14.0 - azure-identity-cpp=1.10.0 @@ -30,7 +30,7 @@ dependencies: - azure-storage-files-datalake-cpp=12.12.0 - black=24.10.0 - blosc=1.21.6 - - bokeh=3.6.0 + - bokeh=3.6.1 - brotli=1.1.0 - brotli-bin=1.1.0 - brotli-python=1.1.0 @@ -52,14 +52,14 @@ dependencies: - cycler=0.12.1 - cyrus-sasl=2.1.27 - cytoolz=1.0.0 - - dask=2024.9.1 - - dask-core=2024.9.1 - - dask-expr=1.1.15 + - dask=2024.10.0 + - dask-core=2024.10.0 + - dask-expr=1.1.16 - dbus=1.13.6 - decorator=5.1.1 - dill=0.3.9 - distlib=0.3.9 - - distributed=2024.9.1 + - distributed=2024.10.0 - docutils=0.21.2 - double-conversion=3.3.0 - earthkit-data=0.5.6 @@ -67,7 +67,7 @@ dependencies: - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 - - expat=2.6.3 + - expat=2.6.4 - filelock=3.16.1 - findlibs=0.0.5 - flake8=7.1.1 @@ -76,13 +76,13 @@ dependencies: - font-ttf-inconsolata=3.000 - font-ttf-source-code-pro=2.038 - font-ttf-ubuntu=0.83 - - fontconfig=2.14.2 + - fontconfig=2.15.0 - fonts-conda-ecosystem=1 - fonts-conda-forge=1 - fonttools=4.54.1 - freeglut=3.2.2 - freetype=2.12.1 - - fsspec=2024.9.0 + - fsspec=2024.10.0 - gflags=2.2.2 - glog=0.7.1 - graphite2=1.3.13 @@ -99,7 +99,7 @@ dependencies: - importlib_metadata=8.5.0 - iniconfig=2.0.0 - ipdb=0.13.13 - - ipython=8.28.0 + - ipython=8.29.0 - isort=5.13.2 - jasper=4.2.4 - jedi=0.19.1 @@ -112,27 +112,27 @@ dependencies: - lerc=4.0.0 - libabseil=20240722.0 - libaec=1.1.3 - - libarrow=17.0.0 - - libarrow-acero=17.0.0 - - libarrow-dataset=17.0.0 - - libarrow-substrait=17.0.0 + - libarrow=18.0.0 + - libarrow-acero=18.0.0 + - libarrow-dataset=18.0.0 + - libarrow-substrait=18.0.0 - libblas=3.9.0 - libbrotlicommon=1.1.0 - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 - - libclang-cpp19.1=19.1.1 - - libclang13=19.1.1 + - libclang-cpp19.1=19.1.3 + - libclang13=19.1.3 - libcrc32c=1.1.2 - libcups=2.3.3 - - libcurl=8.10.1 + - libcurl=8.11.0 - libdeflate=1.22 - libdrm=2.4.123 - libedit=3.1.20191231 - libegl=1.7.0 - libev=4.33 - libevent=2.1.12 - - libexpat=2.6.3 + - libexpat=2.6.4 - libffi=3.4.2 - libgcc=14.2.0 - libgcc-ng=14.2.0 @@ -140,31 +140,31 @@ dependencies: - libgfortran-ng=14.2.0 - libgfortran5=14.2.0 - libgl=1.7.0 - - libglib=2.82.1 + - libglib=2.82.2 - libglu=9.0.0 - libglvnd=1.7.0 - libglx=1.7.0 - libgomp=14.2.0 - libgoogle-cloud=2.30.0 - libgoogle-cloud-storage=2.30.0 - - libgrpc=1.65.5 + - libgrpc=1.67.1 - libiconv=1.17 - libjpeg-turbo=3.0.0 - liblapack=3.9.0 - - libllvm19=19.1.2 + - libllvm19=19.1.3 - libnetcdf=4.9.2 - - libnghttp2=1.58.0 + - libnghttp2=1.64.0 - libnsl=2.0.1 - libntlm=1.4 - - libopenblas=0.3.27 + - libopenblas=0.3.28 - libopengl=1.7.0 - - libparquet=17.0.0 + - libparquet=18.0.0 - libpciaccess=0.18 - libpng=1.6.44 - libpq=17.0 - - libprotobuf=5.27.5 + - libprotobuf=5.28.2 - libre2-11=2024.07.02 - - libsqlite=3.46.1 + - libsqlite=3.47.0 - libssh2=1.11.0 - libstdcxx=14.2.0 - libstdcxx-ng=14.2.0 @@ -175,15 +175,15 @@ dependencies: - libwebp-base=1.4.0 - libxcb=1.17.0 - libxkbcommon=1.7.0 - - libxml2=2.12.7 + - libxml2=2.13.4 - libxslt=1.1.39 - - libzip=1.11.1 + - libzip=1.11.2 - libzlib=1.3.1 - locket=1.0.0 - lz4=4.3.3 - lz4-c=1.9.4 - markdown-it-py=3.0.0 - - markupsafe=3.0.1 + - markupsafe=3.0.2 - matplotlib=3.9.2 - matplotlib-base=3.9.2 - matplotlib-inline=0.1.7 @@ -191,14 +191,14 @@ dependencies: - mdurl=0.1.2 - msgpack-python=1.1.0 - munkres=1.1.4 - - mypy=1.12.0 + - mypy=1.13.0 - mypy_extensions=1.0.0 - mysql-common=9.0.1 - mysql-libs=9.0.1 - ncurses=6.5 - netcdf4=1.7.1 - nodeenv=1.9.1 - - numpy=2.1.2 + - numpy=2.1.3 - openjpeg=2.5.2 - openldap=2.6.8 - openssl=3.3.2 @@ -213,21 +213,20 @@ dependencies: - pdbufr=0.11.0 - pexpect=4.9.0 - pickleshare=0.7.5 - - pillow=10.4.0 - - pip=24.2 + - pillow=11.0.0 + - pip=24.3.1 - pixman=0.43.2 - platformdirs=4.3.6 - pluggy=1.5.0 - pre-commit=4.0.1 - pre-commit-hooks=5.0.0 - prompt-toolkit=3.0.48 - - psutil=6.0.0 + - psutil=6.1.0 - pthread-stubs=0.4 - ptyprocess=0.7.0 - pure_eval=0.2.3 - - pyarrow=17.0.0 - - pyarrow-core=17.0.0 - - pyarrow-hotfix=0.6 + - pyarrow=18.0.0 + - pyarrow-core=18.0.0 - pycodestyle=2.12.1 - pycparser=2.22 - pydantic=2.9.2 @@ -237,7 +236,7 @@ dependencies: - pygments=2.18.0 - pylint=3.3.1 - pyparsing=3.2.0 - - pyside6=6.8.0 + - pyside6=6.8.0.2 - pysocks=1.7.1 - pytest=8.3.3 - python=3.10.8 @@ -252,17 +251,17 @@ dependencies: - qt6-main=6.8.0 - re2=2024.07.02 - readline=8.2 - - regex=2024.9.11 + - regex=2024.11.6 - requests=2.32.3 - - rich=13.9.2 + - rich=13.9.4 - rope=1.13.0 - rstcheck=6.2.4 - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 - ruamel.yaml.clib=0.2.8 - - s2n=1.5.5 + - s2n=1.5.6 - scipy=1.14.1 - - setuptools=75.1.0 + - setuptools=75.3.0 - shellingham=1.5.4 - six=1.16.0 - snappy=1.2.1 @@ -276,7 +275,7 @@ dependencies: - tomlkit=0.13.2 - toolz=1.0.0 - tornado=6.4.1 - - tqdm=4.66.5 + - tqdm=4.67.0 - traitlets=5.14.3 - typer=0.12.5 - typer-slim=0.12.5 @@ -287,11 +286,11 @@ dependencies: - ukkonen=1.0.1 - unicodedata2=15.1.0 - urllib3=2.2.3 - - virtualenv=20.26.6 + - virtualenv=20.27.1 - wayland=1.23.1 - wcwidth=0.2.13 - wheel=0.44.0 - - xarray=2024.9.0 + - xarray=2024.10.0 - xcb-util=0.4.1 - xcb-util-cursor=0.1.5 - xcb-util-image=0.4.0 @@ -304,7 +303,7 @@ dependencies: - xorg-libx11=1.8.10 - xorg-libxau=1.0.11 - xorg-libxcomposite=0.4.6 - - xorg-libxcursor=1.2.2 + - xorg-libxcursor=1.2.3 - xorg-libxdamage=1.1.6 - xorg-libxdmcp=1.1.5 - xorg-libxext=1.3.6 @@ -325,6 +324,6 @@ dependencies: - zstandard=0.23.0 - zstd=1.5.6 - pip: - - coverage==7.6.3 + - coverage==7.6.4 - flake8-pyproject==1.2.3 - - pytest-cov==5.0.0 + - pytest-cov==6.0.0 From 68ee283ae694398016d686b25e83c695290eb8de Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 7 Nov 2024 15:04:12 +0100 Subject: [PATCH 56/62] rename var --- tests/engine/test_stats.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index e35b1a8c..5e32b12c 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -63,7 +63,7 @@ def add_variable_to_grib(filename, dict_data): @pytest.fixture def setup_grib_file(tmp_path): - array_t = np.ones( + array_v = np.ones( ( TIME_DIM_GRIB_SIZE, STEP_DIM_SIZE, @@ -71,10 +71,10 @@ def setup_grib_file(tmp_path): HORIZONTAL_DIM_GRIB_SIZE, ) ) - array_t[:, :, :, 0] = 0 - array_t[:, :, :, -1] = 2 + array_v[:, :, :, 0] = 0 + array_v[:, :, :, -1] = 2 - array_pres = ( + array_t = ( np.ones( ( TIME_DIM_GRIB_SIZE, @@ -85,10 +85,10 @@ def setup_grib_file(tmp_path): ) * 3 ) - array_pres[:, :, :, 0] = 2 - array_pres[:, :, :, -1] = 4 + array_t[:, :, :, 0] = 2 + array_t[:, :, :, -1] = 4 - dict_data = {"t": array_pres, "v": array_t} + dict_data = {"t": array_t, "v": array_v} # This would be where your grib file is created add_variable_to_grib(tmp_path / GRIB_FILENAME, dict_data) From 0a2fc582013e94b110e20a2a93e9aa8116fb78d4 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Thu, 7 Nov 2024 15:28:59 +0100 Subject: [PATCH 57/62] fix name --- tests/engine/test_stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 5e32b12c..809a4da5 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -21,7 +21,7 @@ HORIZONTAL_DIM_GRIB_SIZE = 6114 GRIB_FILENAME = "test_stats_grib.grib" -STATS_FILE_NAMES = "test_stats.csv" +STATS_FILE_NAME = "test_stats.csv" NC_FILE_NAME = "test_stats.nc" NC_FILE_GLOB = "test_s*.nc" @@ -109,7 +109,7 @@ def test_stats_grib(tmp_path): df = create_stats_dataframe( input_dir=str(tmp_path), file_id=[["Test data", GRIB_FILENAME]], - stats_file_name=tmp_path / STATS_FILE_NAMES, + stats_file_name=tmp_path / STATS_FILE_NAME, file_specification=file_specification, ) @@ -173,7 +173,7 @@ def test_stats_netcdf(setup_netcdf_file, tmp_path): # pylint: disable=unused-ar df = create_stats_dataframe( input_dir=str(tmp_path), file_id=[["Test data", NC_FILE_GLOB]], - stats_file_name=tmp_path / STATS_FILE_NAMES, + stats_file_name=tmp_path / STATS_FILE_NAME, file_specification=file_specification, ) From 282a9d37044cd7f88390d98dbad08bd739e37e01 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 8 Nov 2024 15:05:10 +0100 Subject: [PATCH 58/62] rm mamba option --- setup_env.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/setup_env.sh b/setup_env.sh index 45a0fe36..227c3f22 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -10,23 +10,22 @@ fi # Default env names DEFAULT_ENV_NAME="probtest" +CONDA=conda # Default options ENV_NAME="${DEFAULT_ENV_NAME}" PYVERSION=3.10.8 PINNED=true EXPORT=false -CONDA=conda HELP=false -help_msg="Usage: $(basename "${0}") [-n NAME] [-p VER] [-u] [-e] [-m] [-h] +help_msg="Usage: $(basename "${0}") [-n NAME] [-p VER] [-u] [-e] [-h] Options: -n NAME Env name [default: ${DEFAULT_ENV_NAME} -p VER Python version [default: ${PYVERSION}] -u Use unpinned requirements (minimal version restrictions) -e Export environment files (requires -u) - -m Use mamba instead of conda -h Print this help message and exit " @@ -37,7 +36,6 @@ while getopts n:p:defhimu flag; do p) PYVERSION=${OPTARG};; e) EXPORT=true;; h) HELP=true;; - m) CONDA=mamba;; u) PINNED=false;; ?) echo -e "\n${help_msg}" >&2; exit 1;; esac From 937184e4e2c0d59bfda8d2fc8016d83edc6d853c Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 8 Nov 2024 15:05:30 +0100 Subject: [PATCH 59/62] fix name --- tests/engine/test_stats.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/engine/test_stats.py b/tests/engine/test_stats.py index 809a4da5..0893d73c 100644 --- a/tests/engine/test_stats.py +++ b/tests/engine/test_stats.py @@ -20,7 +20,7 @@ HEIGHT_DIM_GRIB_SIZE = 1 HORIZONTAL_DIM_GRIB_SIZE = 6114 -GRIB_FILENAME = "test_stats_grib.grib" +GRIB_FILE_NAME = "test_stats_grib.grib" STATS_FILE_NAME = "test_stats.csv" NC_FILE_NAME = "test_stats.nc" NC_FILE_GLOB = "test_s*.nc" @@ -91,7 +91,7 @@ def setup_grib_file(tmp_path): dict_data = {"t": array_t, "v": array_v} # This would be where your grib file is created - add_variable_to_grib(tmp_path / GRIB_FILENAME, dict_data) + add_variable_to_grib(tmp_path / GRIB_FILE_NAME, dict_data) @pytest.mark.usefixtures("setup_grib_file") @@ -108,7 +108,7 @@ def test_stats_grib(tmp_path): df = create_stats_dataframe( input_dir=str(tmp_path), - file_id=[["Test data", GRIB_FILENAME]], + file_id=[["Test data", GRIB_FILE_NAME]], stats_file_name=tmp_path / STATS_FILE_NAME, file_specification=file_specification, ) From 4321bbce3c0b3163dd1f3dec5bf7c2bea4f69a94 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 8 Nov 2024 15:30:17 +0100 Subject: [PATCH 60/62] update requirements --- requirements/environment.yml | 21 +++++++++++++++------ requirements/requirements.yml | 2 +- setup_env.sh | 2 +- 3 files changed, 17 insertions(+), 8 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index d9e72f30..c0684306 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -62,8 +62,9 @@ dependencies: - distributed=2024.10.0 - docutils=0.21.2 - double-conversion=3.3.0 - - earthkit-data=0.5.6 - - eccodes=2.33.0 + - earthkit-data=0.10.9 + - earthkit-geo=0.2.0 + - eccodes=2.35.0 - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 @@ -97,6 +98,7 @@ dependencies: - idna=3.10 - importlib-metadata=8.5.0 - importlib_metadata=8.5.0 + - importlib_resources=6.4.5 - iniconfig=2.0.0 - ipdb=0.13.13 - ipython=8.29.0 @@ -104,6 +106,8 @@ dependencies: - jasper=4.2.4 - jedi=0.19.1 - jinja2=3.1.4 + - jsonschema=4.23.0 + - jsonschema-specifications=2024.10.1 - keyutils=1.6.1 - kiwisolver=1.4.7 - krb5=1.21.3 @@ -125,7 +129,7 @@ dependencies: - libclang13=19.1.3 - libcrc32c=1.1.2 - libcups=2.3.3 - - libcurl=8.11.0 + - libcurl=8.10.1 - libdeflate=1.22 - libdrm=2.4.123 - libedit=3.1.20191231 @@ -180,8 +184,10 @@ dependencies: - libzip=1.11.2 - libzlib=1.3.1 - locket=1.0.0 + - lru-dict=1.3.0 - lz4=4.3.3 - lz4-c=1.9.4 + - markdown=3.6 - markdown-it-py=3.0.0 - markupsafe=3.0.2 - matplotlib=3.9.2 @@ -216,6 +222,7 @@ dependencies: - pillow=11.0.0 - pip=24.3.1 - pixman=0.43.2 + - pkgutil-resolve-name=1.3.10 - platformdirs=4.3.6 - pluggy=1.5.0 - pre-commit=4.0.1 @@ -251,10 +258,12 @@ dependencies: - qt6-main=6.8.0 - re2=2024.07.02 - readline=8.2 + - referencing=0.35.1 - regex=2024.11.6 - requests=2.32.3 - rich=13.9.4 - rope=1.13.0 + - rpds-py=0.21.0 - rstcheck=6.2.4 - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 @@ -277,9 +286,9 @@ dependencies: - tornado=6.4.1 - tqdm=4.67.0 - traitlets=5.14.3 - - typer=0.12.5 - - typer-slim=0.12.5 - - typer-slim-standard=0.12.5 + - typer=0.13.0 + - typer-slim=0.13.0 + - typer-slim-standard=0.13.0 - typing-extensions=4.12.2 - typing_extensions=4.12.2 - tzdata=2024b diff --git a/requirements/requirements.yml b/requirements/requirements.yml index 49a8ea58..260339ad 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -7,7 +7,7 @@ dependencies: # runtime - click>=7.1.2 - earthkit-data - - eccodes==2.33.0 + - eccodes==2.35.0 - Jinja2>=3.0.1 - matplotlib>=3.2.1 - netCDF4>=1.5.3 diff --git a/setup_env.sh b/setup_env.sh index 227c3f22..cbeb5efd 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -72,7 +72,7 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEFINITION_VERSION="v2.33.0.5d" +DEFINITION_VERSION="v2.35.0.1d" DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} From 8ca9638c9085b6c9e4b122cca64967541f7a2698 Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 22 Nov 2024 10:53:00 +0100 Subject: [PATCH 61/62] update eccodes, needs test fixing --- requirements/environment.yml | 98 +++++++++++++++++------------------ requirements/requirements.yml | 3 +- setup_env.sh | 2 +- 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index c0684306..2c06d2b7 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -5,24 +5,25 @@ channels: dependencies: - _libgcc_mutex=0.1 - _openmp_mutex=4.5 - - alsa-lib=1.2.12 + - alsa-lib=1.2.13 - annotated-types=0.7.0 + - array-api-compat=1.9.1 - astroid=3.3.5 - asttokens=2.4.1 - attrs=24.2.0 - aws-c-auth=0.8.0 - aws-c-cal=0.8.0 - - aws-c-common=0.10.0 + - aws-c-common=0.10.3 - aws-c-compression=0.3.0 - aws-c-event-stream=0.5.0 - - aws-c-http=0.9.0 - - aws-c-io=0.15.0 + - aws-c-http=0.9.1 + - aws-c-io=0.15.2 - aws-c-mqtt=0.11.0 - - aws-c-s3=0.7.0 - - aws-c-sdkutils=0.2.0 - - aws-checksums=0.2.0 - - aws-crt-cpp=0.29.1 - - aws-sdk-cpp=1.11.407 + - aws-c-s3=0.7.1 + - aws-c-sdkutils=0.2.1 + - aws-checksums=0.2.2 + - aws-crt-cpp=0.29.5 + - aws-sdk-cpp=1.11.449 - azure-core-cpp=1.14.0 - azure-identity-cpp=1.10.0 - azure-storage-blobs-cpp=12.13.0 @@ -35,7 +36,7 @@ dependencies: - brotli-bin=1.1.0 - brotli-python=1.1.0 - bzip2=1.0.8 - - c-ares=1.34.2 + - c-ares=1.34.3 - ca-certificates=2024.8.30 - cairo=1.18.0 - certifi=2024.8.30 @@ -48,23 +49,23 @@ dependencies: - cloudpickle=3.1.0 - codespell=2.3.0 - colorama=0.4.6 - - contourpy=1.3.0 + - contourpy=1.3.1 - cycler=0.12.1 - cyrus-sasl=2.1.27 - cytoolz=1.0.0 - - dask=2024.10.0 - - dask-core=2024.10.0 - - dask-expr=1.1.16 + - dask=2024.11.2 + - dask-core=2024.11.2 + - dask-expr=1.1.19 - dbus=1.13.6 - decorator=5.1.1 - dill=0.3.9 - distlib=0.3.9 - - distributed=2024.10.0 + - distributed=2024.11.2 - docutils=0.21.2 - double-conversion=3.3.0 - - earthkit-data=0.10.9 - - earthkit-geo=0.2.0 - - eccodes=2.35.0 + - earthkit-data=0.11.1 + - earthkit-geo=0.3.0 + - eccodes=2.38.3 - entrypoints=0.4 - exceptiongroup=1.2.2 - executing=2.1.0 @@ -80,7 +81,7 @@ dependencies: - fontconfig=2.15.0 - fonts-conda-ecosystem=1 - fonts-conda-forge=1 - - fonttools=4.54.1 + - fonttools=4.55.0 - freeglut=3.2.2 - freetype=2.12.1 - fsspec=2024.10.0 @@ -90,21 +91,20 @@ dependencies: - h2=4.1.0 - harfbuzz=9.0.0 - hdf4=4.2.15 - - hdf5=1.14.3 + - hdf5=1.14.4 - hpack=4.0.0 - hyperframe=6.0.1 - icu=75.1 - - identify=2.6.1 + - identify=2.6.2 - idna=3.10 - importlib-metadata=8.5.0 - - importlib_metadata=8.5.0 - importlib_resources=6.4.5 - iniconfig=2.0.0 - ipdb=0.13.13 - ipython=8.29.0 - isort=5.13.2 - jasper=4.2.4 - - jedi=0.19.1 + - jedi=0.19.2 - jinja2=3.1.4 - jsonschema=4.23.0 - jsonschema-specifications=2024.10.1 @@ -125,8 +125,8 @@ dependencies: - libbrotlidec=1.1.0 - libbrotlienc=1.1.0 - libcblas=3.9.0 - - libclang-cpp19.1=19.1.3 - - libclang13=19.1.3 + - libclang-cpp19.1=19.1.4 + - libclang13=19.1.4 - libcrc32c=1.1.2 - libcups=2.3.3 - libcurl=8.10.1 @@ -141,21 +141,20 @@ dependencies: - libgcc=14.2.0 - libgcc-ng=14.2.0 - libgfortran=14.2.0 - - libgfortran-ng=14.2.0 - libgfortran5=14.2.0 - libgl=1.7.0 - libglib=2.82.2 - - libglu=9.0.0 + - libglu=9.0.3 - libglvnd=1.7.0 - libglx=1.7.0 - libgomp=14.2.0 - - libgoogle-cloud=2.30.0 - - libgoogle-cloud-storage=2.30.0 + - libgoogle-cloud=2.31.0 + - libgoogle-cloud-storage=2.31.0 - libgrpc=1.67.1 - libiconv=1.17 - libjpeg-turbo=3.0.0 - liblapack=3.9.0 - - libllvm19=19.1.3 + - libllvm19=19.1.4 - libnetcdf=4.9.2 - libnghttp2=1.64.0 - libnsl=2.0.1 @@ -165,7 +164,7 @@ dependencies: - libparquet=18.0.0 - libpciaccess=0.18 - libpng=1.6.44 - - libpq=17.0 + - libpq=17.2 - libprotobuf=5.28.2 - libre2-11=2024.07.02 - libsqlite=3.47.0 @@ -179,7 +178,7 @@ dependencies: - libwebp-base=1.4.0 - libxcb=1.17.0 - libxkbcommon=1.7.0 - - libxml2=2.13.4 + - libxml2=2.13.5 - libxslt=1.1.39 - libzip=1.11.2 - libzlib=1.3.1 @@ -202,14 +201,14 @@ dependencies: - mysql-common=9.0.1 - mysql-libs=9.0.1 - ncurses=6.5 - - netcdf4=1.7.1 + - netcdf4=1.7.2 - nodeenv=1.9.1 - numpy=2.1.3 - openjpeg=2.5.2 - openldap=2.6.8 - - openssl=3.3.2 - - orc=2.0.2 - - packaging=24.1 + - openssl=3.4.0 + - orc=2.0.3 + - packaging=24.2 - pandas=2.2.3 - parso=0.8.4 - partd=1.4.2 @@ -236,8 +235,8 @@ dependencies: - pyarrow-core=18.0.0 - pycodestyle=2.12.1 - pycparser=2.22 - - pydantic=2.9.2 - - pydantic-core=2.23.4 + - pydantic=2.10.0 + - pydantic-core=2.27.0 - pydocstyle=6.3.0 - pyflakes=3.2.0 - pygments=2.18.0 @@ -247,8 +246,8 @@ dependencies: - pysocks=1.7.1 - pytest=8.3.3 - python=3.10.8 - - python-dateutil=2.9.0 - - python-eccodes=1.7.1 + - python-dateutil=2.9.0.post0 + - python-eccodes=2.37.0 - python-tzdata=2024.2 - python_abi=3.10 - pytoolconfig=1.2.5 @@ -268,9 +267,9 @@ dependencies: - rstcheck-core=1.2.1 - ruamel.yaml=0.18.6 - ruamel.yaml.clib=0.2.8 - - s2n=1.5.6 + - s2n=1.5.9 - scipy=1.14.1 - - setuptools=75.3.0 + - setuptools=75.6.0 - shellingham=1.5.4 - six=1.16.0 - snappy=1.2.1 @@ -280,15 +279,15 @@ dependencies: - tblib=3.0.0 - tk=8.6.13 - toml=0.10.2 - - tomli=2.0.2 + - tomli=2.1.0 - tomlkit=0.13.2 - toolz=1.0.0 - tornado=6.4.1 - tqdm=4.67.0 - traitlets=5.14.3 - - typer=0.13.0 - - typer-slim=0.13.0 - - typer-slim-standard=0.13.0 + - typer=0.13.1 + - typer-slim=0.13.1 + - typer-slim-standard=0.13.1 - typing-extensions=4.12.2 - typing_extensions=4.12.2 - tzdata=2024b @@ -298,7 +297,7 @@ dependencies: - virtualenv=20.27.1 - wayland=1.23.1 - wcwidth=0.2.13 - - wheel=0.44.0 + - wheel=0.45.0 - xarray=2024.10.0 - xcb-util=0.4.1 - xcb-util-cursor=0.1.5 @@ -322,17 +321,16 @@ dependencies: - xorg-libxrender=0.9.11 - xorg-libxtst=1.2.5 - xorg-libxxf86vm=1.1.5 - - xorg-xextproto=7.3.0 - xorg-xorgproto=2024.1 - xyzservices=2024.9.0 - xz=5.2.6 - yaml=0.2.5 - zict=3.0.0 - - zipp=3.20.2 + - zipp=3.21.0 - zlib=1.3.1 - zstandard=0.23.0 - zstd=1.5.6 - pip: - - coverage==7.6.4 + - coverage==7.6.7 - flake8-pyproject==1.2.3 - pytest-cov==6.0.0 diff --git a/requirements/requirements.yml b/requirements/requirements.yml index 260339ad..89d21998 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -5,9 +5,10 @@ dependencies: - python==3.10.8 - pip>=22.3 # runtime + - array-api-compat - click>=7.1.2 - earthkit-data - - eccodes==2.35.0 + - eccodes>=2.38.0 - Jinja2>=3.0.1 - matplotlib>=3.2.1 - netCDF4>=1.5.3 diff --git a/setup_env.sh b/setup_env.sh index cbeb5efd..41744f78 100755 --- a/setup_env.sh +++ b/setup_env.sh @@ -72,7 +72,7 @@ fi ${CONDA} activate ${ENV_NAME} CONDA_LOC=${CONDA_PREFIX} -DEFINITION_VERSION="v2.35.0.1d" +DEFINITION_VERSION="v2.36.0.2" DEFINITION_PATH_DEFAULT=${CONDA_LOC}/share/eccodes DEFINITION_PATH_RESOURCES=${CONDA_LOC}/share/eccodes-cosmo-resources_${DEFINITION_VERSION} From a4384606ca21ca651cf611ed579f10c0f91747fa Mon Sep 17 00:00:00 2001 From: Daniel Hupp Date: Fri, 22 Nov 2024 12:42:11 +0100 Subject: [PATCH 62/62] fix environment --- requirements/environment.yml | 2 +- requirements/requirements.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/environment.yml b/requirements/environment.yml index 2c06d2b7..4d975866 100644 --- a/requirements/environment.yml +++ b/requirements/environment.yml @@ -63,7 +63,7 @@ dependencies: - distributed=2024.11.2 - docutils=0.21.2 - double-conversion=3.3.0 - - earthkit-data=0.11.1 + - earthkit-data=0.10.11 - earthkit-geo=0.3.0 - eccodes=2.38.3 - entrypoints=0.4 diff --git a/requirements/requirements.yml b/requirements/requirements.yml index 89d21998..943fcbe8 100644 --- a/requirements/requirements.yml +++ b/requirements/requirements.yml @@ -7,7 +7,7 @@ dependencies: # runtime - array-api-compat - click>=7.1.2 - - earthkit-data + - earthkit-data<0.11 - eccodes>=2.38.0 - Jinja2>=3.0.1 - matplotlib>=3.2.1