From 1c8096f8799a58bfbbacbe0450d31bc27cb6b096 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Sat, 9 Nov 2024 11:44:57 -0500 Subject: [PATCH 1/9] Add state of ground to increase the snow depth reports. --- .../snow/obs/config/bufr2ioda_mapping.yaml.j2 | 1 + parm/snow/obs/config/bufr_sfcsno_mapping.yaml | 17 +++++----- .../obs/config/ioda_bufr_python_encoder.py | 31 +++++++++++++++++++ parm/snow/obs/config/sfcsno_snow.yaml.j2 | 6 ++-- sorc/bufr-query | 2 +- 5 files changed, 47 insertions(+), 10 deletions(-) create mode 100644 parm/snow/obs/config/ioda_bufr_python_encoder.py diff --git a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 index 9340cf507..fd0f1d3f5 100644 --- a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 +++ b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 @@ -2,3 +2,4 @@ mkdir: - '{{ DATA }}/obs' copy: - ['{{PARMgfs}}/gdas/snow/obs/config/bufr_sfcsno_mapping.yaml', '{{ DATA }}/obs/'] +- ['{{PARMgfs}}/gdas/snow/obs/config/ioda_bufr_python_encoder.py', '{{ DATA }}'] diff --git a/parm/snow/obs/config/bufr_sfcsno_mapping.yaml b/parm/snow/obs/config/bufr_sfcsno_mapping.yaml index 00ab1bc7d..2fe12f39c 100644 --- a/parm/snow/obs/config/bufr_sfcsno_mapping.yaml +++ b/parm/snow/obs/config/bufr_sfcsno_mapping.yaml @@ -17,7 +17,6 @@ bufr: query: "[*/CLON, */CLONH]" stationIdentification: query: "*/RPID" - stationElevation: query: "[*/SELV, */HSMSL]" @@ -26,11 +25,8 @@ bufr: query: "[*/SNWSQ1/TOSD, */MTRMSC/TOSD, */STGDSNDM/TOSD]" transforms: - scale: 1000.0 - filters: - - bounding: - variable: totalSnowDepth - lowerBound: 0 - upperBound: 10000000 + groundState: + query: "[*/GRDSQ1/SOGR, */STGDSNDM/SOGR]" encoder: variables: @@ -65,7 +61,7 @@ encoder: coordinates: "longitude latitude" source: variables/stationIdentification longName: "Identification of Observing Location" - units: "m" + units: "index" # ObsValue - name: "ObsValue/totalSnowDepth" @@ -73,3 +69,10 @@ encoder: source: variables/totalSnowDepth longName: "Total Snow Depth" units: "mm" + + - name: "ObsValue/groundState" + coordinates: "longitude latitude" + source: variables/groundState + longName: "STATE OF THE GROUND" + units: "index" + diff --git a/parm/snow/obs/config/ioda_bufr_python_encoder.py b/parm/snow/obs/config/ioda_bufr_python_encoder.py new file mode 100644 index 000000000..98cccf288 --- /dev/null +++ b/parm/snow/obs/config/ioda_bufr_python_encoder.py @@ -0,0 +1,31 @@ +import numpy as np +import bufr +from pyioda.ioda.Engines.Bufr import Encoder + + +def mask_container(container, mask): + new_container = bufr.DataContainer() + for var_name in container.list(): + print(f" ... variable name: {var_name} ...") + var = container.get(var_name) + paths = container.get_paths(var_name) + new_container.add(var_name, var[mask], paths) + + return new_container + +def create_obs_group(input_path): + YAML_PATH = "./obs/bufr_sfcsno_mapping.yaml" + container = bufr.Parser(input_path, YAML_PATH).parse() + + sogr = container.get('variables/groundState') + snod = container.get('variables/totalSnowDepth') + snod[(sogr < 10.0) | (sogr == 11.0) | (sogr == 15.0)] = 0.0 + container.replace('variables/totalSnowDepth', snod) + + print(f" ... Remove filled/missing snow values ...") + masked_container = mask_container(container, (~snod.mask)) + + encoder = Encoder(YAML_PATH) + data = next(iter(encoder.encode(masked_container).values())) + + return data diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 960ea8f64..7fe48e45f 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -5,9 +5,11 @@ halo size: 250e3 obsdatain: engine: - type: bufr + type: script + script file: "{{ DATA }}/ioda_bufr_python_encoder.py" + args: + input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' - mapping file: '{{ DATA }}/obs/bufr_sfcsno_mapping.yaml' obsdataout: engine: type: H5File diff --git a/sorc/bufr-query b/sorc/bufr-query index 97367fcd5..8bc28860d 160000 --- a/sorc/bufr-query +++ b/sorc/bufr-query @@ -1 +1 @@ -Subproject commit 97367fcd59adf4863aba1a52189e20f9f66451af +Subproject commit 8bc28860d562f9a53ffa86e7c5c5ae05c2c4e09f From 365675ddbb7d56d0de04aaa5341e17f572c5628e Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Tue, 12 Nov 2024 19:19:04 -0500 Subject: [PATCH 2/9] Update the defined index with zero snow depth. --- parm/snow/obs/config/ioda_bufr_python_encoder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/snow/obs/config/ioda_bufr_python_encoder.py b/parm/snow/obs/config/ioda_bufr_python_encoder.py index 98cccf288..1132b416f 100644 --- a/parm/snow/obs/config/ioda_bufr_python_encoder.py +++ b/parm/snow/obs/config/ioda_bufr_python_encoder.py @@ -19,7 +19,7 @@ def create_obs_group(input_path): sogr = container.get('variables/groundState') snod = container.get('variables/totalSnowDepth') - snod[(sogr < 10.0) | (sogr == 11.0) | (sogr == 15.0)] = 0.0 + snod[(sogr <= 11.0) | (sogr == 15.0)] = 0.0 container.replace('variables/totalSnowDepth', snod) print(f" ... Remove filled/missing snow values ...") From 0c025221a8b456f6a65d149bccaa93c4e672e3b9 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 13 Nov 2024 12:45:55 -0500 Subject: [PATCH 3/9] Move python code to parm/gdas/snow/ directory to aviod extra copy. --- parm/snow/{obs/config => }/ioda_bufr_python_encoder.py | 0 parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 | 1 - parm/snow/obs/config/sfcsno_snow.yaml.j2 | 2 +- 3 files changed, 1 insertion(+), 2 deletions(-) rename parm/snow/{obs/config => }/ioda_bufr_python_encoder.py (100%) diff --git a/parm/snow/obs/config/ioda_bufr_python_encoder.py b/parm/snow/ioda_bufr_python_encoder.py similarity index 100% rename from parm/snow/obs/config/ioda_bufr_python_encoder.py rename to parm/snow/ioda_bufr_python_encoder.py diff --git a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 index fd0f1d3f5..9340cf507 100644 --- a/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 +++ b/parm/snow/obs/config/bufr2ioda_mapping.yaml.j2 @@ -2,4 +2,3 @@ mkdir: - '{{ DATA }}/obs' copy: - ['{{PARMgfs}}/gdas/snow/obs/config/bufr_sfcsno_mapping.yaml', '{{ DATA }}/obs/'] -- ['{{PARMgfs}}/gdas/snow/obs/config/ioda_bufr_python_encoder.py', '{{ DATA }}'] diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 7fe48e45f..9f2d9cd24 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -6,7 +6,7 @@ obsdatain: engine: type: script - script file: "{{ DATA }}/ioda_bufr_python_encoder.py" + script file: "{{ PARMgfs }}/gdas/snow/ioda_bufr_python_encoder.py" args: input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' From e57ac67561baee301284815a0edc233836bb64a6 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Wed, 13 Nov 2024 18:29:36 -0500 Subject: [PATCH 4/9] Modif to only replace the missing/filled snod values. --- parm/snow/ioda_bufr_python_encoder.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/parm/snow/ioda_bufr_python_encoder.py b/parm/snow/ioda_bufr_python_encoder.py index 1132b416f..f8f9c0f58 100644 --- a/parm/snow/ioda_bufr_python_encoder.py +++ b/parm/snow/ioda_bufr_python_encoder.py @@ -6,7 +6,6 @@ def mask_container(container, mask): new_container = bufr.DataContainer() for var_name in container.list(): - print(f" ... variable name: {var_name} ...") var = container.get(var_name) paths = container.get_paths(var_name) new_container.add(var_name, var[mask], paths) @@ -14,15 +13,28 @@ def mask_container(container, mask): return new_container def create_obs_group(input_path): + """Create the ioda snow observations + This method: + - reads state of ground (sogr) and snow depth (snod) + - applys sogr conditions to the missing snod values + - removes the filled/missing snow values and creates the masked container + - encoders the new container. + + Parameters + ---------- + input_path + The input bufr file + """ + YAML_PATH = "./obs/bufr_sfcsno_mapping.yaml" container = bufr.Parser(input_path, YAML_PATH).parse() sogr = container.get('variables/groundState') snod = container.get('variables/totalSnowDepth') - snod[(sogr <= 11.0) | (sogr == 15.0)] = 0.0 + snod[(sogr <= 11.0) & snod.mask] = 0.0 + snod[(sogr == 15.0) & snod.mask] = 0.0 container.replace('variables/totalSnowDepth', snod) - print(f" ... Remove filled/missing snow values ...") masked_container = mask_container(container, (~snod.mask)) encoder = Encoder(YAML_PATH) From 244693c68d47087bf530cc2d117ad2989e5193d2 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 14 Nov 2024 17:45:49 -0500 Subject: [PATCH 5/9] Address reviewer's comments. --- parm/snow/obs/config/sfcsno_snow.yaml.j2 | 2 +- .../ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) rename parm/snow/ioda_bufr_python_encoder.py => ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py (92%) diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 9f2d9cd24..d17d3e9b0 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -6,7 +6,7 @@ obsdatain: engine: type: script - script file: "{{ PARMgfs }}/gdas/snow/ioda_bufr_python_encoder.py" + script file: "{{ USHgfs }}/bufr2ioda_sfcsno_bufr_encoder.py" args: input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' diff --git a/parm/snow/ioda_bufr_python_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py similarity index 92% rename from parm/snow/ioda_bufr_python_encoder.py rename to ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index f8f9c0f58..9fdc59538 100644 --- a/parm/snow/ioda_bufr_python_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -37,7 +37,6 @@ def create_obs_group(input_path): masked_container = mask_container(container, (~snod.mask)) - encoder = Encoder(YAML_PATH) - data = next(iter(encoder.encode(masked_container).values())) + data = next(iter(Encoder(YAML_PATH).encode(masked_container).values())) return data From 76a5e94be2810d1a4273fe3b04a46a2664641195 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Thu, 14 Nov 2024 17:54:21 -0500 Subject: [PATCH 6/9] Fix py norms error. --- ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index 9fdc59538..16a648acd 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -12,13 +12,14 @@ def mask_container(container, mask): return new_container + def create_obs_group(input_path): """Create the ioda snow observations This method: - reads state of ground (sogr) and snow depth (snod) - applys sogr conditions to the missing snod values - removes the filled/missing snow values and creates the masked container - - encoders the new container. + - encoders the new container. Parameters ---------- From 2a72e4dc15670bc99c3dc9e2e79a114a30bdbb5d Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 15 Nov 2024 12:07:49 -0500 Subject: [PATCH 7/9] Add the mapping_path indicating the script backend requirements. --- parm/snow/obs/config/sfcsno_snow.yaml.j2 | 1 + ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index d17d3e9b0..83cab6f6e 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -9,6 +9,7 @@ script file: "{{ USHgfs }}/bufr2ioda_sfcsno_bufr_encoder.py" args: input_path: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' + mapping_path: '{{ DATA }}/obs/bufr_sfcsno_mapping.yaml' obsfile: '{{ DATA }}/obs/{{ OPREFIX }}sfcsno.tm00.bufr_d' obsdataout: engine: diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index 16a648acd..140f508f1 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -13,7 +13,7 @@ def mask_container(container, mask): return new_container -def create_obs_group(input_path): +def create_obs_group(input_path, mapping_path): """Create the ioda snow observations This method: - reads state of ground (sogr) and snow depth (snod) @@ -27,7 +27,7 @@ def create_obs_group(input_path): The input bufr file """ - YAML_PATH = "./obs/bufr_sfcsno_mapping.yaml" + YAML_PATH = mapping_path container = bufr.Parser(input_path, YAML_PATH).parse() sogr = container.get('variables/groundState') From fbd1bb960b94bb5f0d6674a0a7860dbbc0816f51 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 15 Nov 2024 12:14:44 -0500 Subject: [PATCH 8/9] Add comments --- ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py index 140f508f1..6c75df0ae 100644 --- a/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py +++ b/ush/ioda/bufr2ioda/bufr2ioda_sfcsno_bufr_encoder.py @@ -23,8 +23,9 @@ def create_obs_group(input_path, mapping_path): Parameters ---------- - input_path - The input bufr file + input_path: The input bufr file + mapping_path: The input bufr2ioda mapping file + """ YAML_PATH = mapping_path From 91f160e614f8abef21529a9611f8aab6105f0ab3 Mon Sep 17 00:00:00 2001 From: Jiarui Dong Date: Fri, 22 Nov 2024 15:36:13 -0500 Subject: [PATCH 9/9] Update the missing value for elevation from bufr file. --- parm/snow/obs/config/sfcsno_snow.yaml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parm/snow/obs/config/sfcsno_snow.yaml.j2 b/parm/snow/obs/config/sfcsno_snow.yaml.j2 index 83cab6f6e..947d6340e 100644 --- a/parm/snow/obs/config/sfcsno_snow.yaml.j2 +++ b/parm/snow/obs/config/sfcsno_snow.yaml.j2 @@ -57,7 +57,7 @@ where: - variable: name: MetaData/stationElevation - minvalue: -999.0 + maxvalue: 999999999.9 - filter: Domain Check # land only where: - variable: