Add reading grib files for stats

.gitignore

@@ -1,3 +1,4 @@
+.coverage
 .idea*
 .vscode
 *pycache*
@@ -6,12 +7,9 @@
 *.png
 *.log
 *.nc
-.coverage
 coverage.xml
 miniconda
 
-# output from unittest
-test_stats.csv
-test_stats_csv.csv
-test_stats_csv.dat
+# output from pytest
 random_tolerance.csv
+random_tolerance_None.csv

AUTHORS.md

@@ -6,6 +6,7 @@
 | Marek Jacob             | DWD                                                 |
 | Jonas Jucker            | C2SM                                                |
 | Annika Lauber           | C2SM                                                |
+| Andrea Leuthard         | MeteoSwiss                                          |
 | Giacomo Serafini        | MeteoSwiss                                          |
 | Mikael Stellio          | C2SM                                                |
 | Ben Weber               | MeteoSwiss                                          |

README.md

@@ -100,10 +100,13 @@ The pinned requirements can be installed by
 ```console
 ./setup_env.sh
 ```
+which is recommended for users.
+
 The unpinned requirements and updating the environment can be done by
 ```console
 ./setup_env.sh -u -e
 ```
+which is recommended for developers and required for adding new requirements.
 
 
 ### The init command
@@ -138,7 +141,17 @@ here should refer to your experiment script:
 
 ```console
 cd icon-base-dir/reference-build
-python ../externals/probtest/probtest.py init --codebase-install $PWD --experiment-name exp_name --reference $PWD --file-id NetCDF "*atm_3d_ml*.nc" --file-id NetCDF "*atm_3d_il*.nc" --file-id NetCDF "*atm_3d_hl*.nc" --file-id NetCDF "*atm_3d_pl*.nc" --file-id latlon "*atm_2d_ll*.nc" --file-id meteogram "Meteogram*.nc"
+python ../externals/probtest/probtest.py init  \
+            --codebase-install $PWD  \
+            --experiment-name exp_name  \
+            --reference $PWD  \
+            --file-id NetCDF "*atm_3d_ml*.nc"  \
+            --file-id NetCDF "*atm_3d_il*.nc"  \
+            --file-id NetCDF "*atm_3d_hl*.nc"  \
+            --file-id NetCDF "*atm_3d_pl*.nc"  \
+            --file-id latlon "*atm_2d_ll*.nc"  \
+            --file-id meteogram "Meteogram*.nc"  \
+            --file-id GRIB "lfff0*z"
 ```
 You might need to update the used account in the json file.
 The perturbation amplitude may also need to be changed in the json file

engine/cdo_table.py

@@ -17,7 +17,7 @@
 from util.click_util import CommaSeperatedInts, cli_help
 from util.constants import cdo_bins
 from util.dataframe_ops import df_from_file_ids
-from util.file_system import file_names_from_pattern
+from util.file_system import get_file_names_from_pattern
 from util.log_handler import logger
 
 
@@ -139,21 +139,21 @@ def cdo_table(
     assert isinstance(file_specification, dict), "must be dict"
 
     # save original method and restore at the end of this module
-    dataframe_from_ncfile_orig = model_output_parser.dataframe_from_ncfile
+    dataframe_from_ncfile_orig = model_output_parser.create_statistics_dataframe
     # modify netcdf parse method:
-    model_output_parser.dataframe_from_ncfile = rel_diff_stats
+    model_output_parser.create_statistics_dataframe = rel_diff_stats
 
     # step 1: compute rel-diff netcdf files
     with tempfile.TemporaryDirectory() as tmpdir:
         for _, file_pattern in file_id:
-            ref_files, err = file_names_from_pattern(model_output_dir, file_pattern)
+            ref_files, err = get_file_names_from_pattern(model_output_dir, file_pattern)
             if err > 0:
                 logger.info(
                     "did not find any files for pattern %s. Continue.", file_pattern
                 )
                 continue
             ref_files.sort()
-            perturb_files, err = file_names_from_pattern(
+            perturb_files, err = get_file_names_from_pattern(
                 perturbed_model_output_dir.format(member_id=member_id), file_pattern
             )
             if err > 0:
@@ -202,4 +202,4 @@ def cdo_table(
 
         Path(cdo_table_file).parent.mkdir(parents=True, exist_ok=True)
         df.to_csv(cdo_table_file)
-        model_output_parser.dataframe_from_ncfile = dataframe_from_ncfile_orig
+        model_output_parser.create_statistics_dataframe = dataframe_from_ncfile_orig

engine/performance.py

@@ -15,7 +15,7 @@
 import click
 
 from util.click_util import cli_help
-from util.file_system import file_names_from_pattern
+from util.file_system import get_file_names_from_pattern
 from util.icon.extract_timings import read_logfile
 from util.log_handler import logger
 from util.tree import TREEFILE_TEMPLATE, TimingTree
@@ -36,7 +36,9 @@ def performance(timing_regex, timing_database, append_time):
     if timing_file_name_base == "":
         timing_file_name_base = "."
 
-    timing_file_name, err = file_names_from_pattern(timing_file_name_base, timing_regex)
+    timing_file_name, err = get_file_names_from_pattern(
+        timing_file_name_base, timing_regex
+    )
     if err > 0:
         logger.info("Did not find any files for regex %s", timing_regex)
         sys.exit(1)