ENH: use of of os.path.join for path creation in get_file_list

doc/source/overview/main.rst

@@ -11,7 +11,7 @@ name                  STRING   Name of the data processing. This name is used in
                                ``<saveimgbasepath>/<name>/<YYYY-MM-DD>/<datasetname>/<prodname>/<outputname>``
 datapath              STRING   Base directory of the rainbow raw data. This field must have a trailing '/'. The raw data files of a scan can be found using the following file path:
                                ``<datapath>/<scanname>/<YYYY-MM-DD>/<YYYYMMDDHHMMSS00datatype>.<ext>``
-configpath            STRING   Base directory of the configuration files. This directory contains clutter maps, filter coefficients, antenna pattern, and the data processing configuration files.
+configpath            STRING   Absolute base directory of the configuration files. This directory contains clutter maps, filter coefficients, antenna pattern, and the data processing configuration files. If any path within a pyrad config file is provided as a relative path, it will be converted to an absolute path, by prepending it with the path specified with "configpath".
 locationConfigFile    STRING   File name (with full path) of the location configuration file. Described in Section 3.2.
 productConfigFile     STRING   File name (with full path) of the product configuration file. Described in Section 4.
 lastStateFile         STRING   File name (with full path) of the file containing the time of the last processed scan. Used in particular for real-time processing.

src/pyrad_proc/pyrad/flow/flow_aux.py

@@ -1038,6 +1038,15 @@ def _create_cfg_dict(cfgfile):
     try:
         print("- Main config file : {}".format(cfgfile))
         cfg = read_config(cfg["configFile"], cfg=cfg, defaults=DEFAULT_CONFIG["main"])
+
+        # Convert loc and prod config files to absolute paths if needed
+        filenames = ["locationConfigFile", 
+        "productConfigFile",
+        ]
+        for fname in filenames:
+            if not os.path.isabs(cfg[fname]):
+                cfg[fname] = os.path.join(cfg['configpath'], cfg[fname])
+
         print("- Location config file : {}".format(cfg["locationConfigFile"]))
         cfg = read_config(
             cfg["locationConfigFile"], cfg=cfg, defaults=DEFAULT_CONFIG["loc"]
@@ -1172,6 +1181,30 @@ def _create_cfg_dict(cfgfile):
         if param in cfg and isinstance(cfg[param], float):
             cfg[param] = [cfg[param]]
 
+    # check whether specified paths are relative or absolute
+    # if relative add configpath to the path
+    filenames = ["locationConfigFile", 
+        "productConfigFile",
+        "datapath",
+        "iconpath",
+        "dempath",
+        "gecsxbasepath",
+        "gecsxname",
+        "loadbasepath",
+        "psrpath",
+        "iqpath",
+        "satpath",
+        "smnpath"]
+
+    for fname in filenames:
+        if type(cfg[fname]) == list:
+            for val in cfg[fname]:
+                if not os.path.isabs(val):
+                    val = os.path.join(cfg['configpath'], val)
+        elif type(cfg[fname]) == str:
+            if not os.path.isabs(cfg[fname]):
+                cfg[fname] = os.path.join(cfg['configpath'], cfg[fname])
+
     # if specified in config, convert coordinates to arrays
     if "RadarPosition" in cfg:
         fltarr_list = ["latitude", "longitude", "altitude"]

src/pyrad_proc/pyrad/io/read_data_radar.py

@@ -4062,39 +4062,38 @@ def merge_scans_iq_rad4alp(
 
 def merge_fields_rainbow(basepath, scan_name, voltime, datatype_list):
     """
-    merge Rainbow fields into a single radar object.
+    Merge Rainbow fields into a single radar object.
 
     Parameters
     ----------
     basepath : str
-        name of the base path where to find the data
+        Name of the base path where to find the data.
     scan_name: str
-        name of the scan
+        Name of the scan.
     voltime : datetime object
-        reference time of the scan
+        Reference time of the scan.
     datatype_list : list
-        lists of data types to get
+        Lists of data types to get.
 
     Returns
     -------
     radar : Radar
-        radar object
-
+        Radar object.
     """
-    datapath = basepath + scan_name + voltime.strftime("%Y-%m-%d") + "/"
+    datapath = os.path.join(basepath, scan_name, voltime.strftime("%Y-%m-%d") + "/")
     fdatetime = voltime.strftime("%Y%m%d%H%M%S") + "00"
 
     if (datatype_list[0] != "Nh") and (datatype_list[0] != "Nv"):
-        filename = glob.glob(datapath + fdatetime + datatype_list[0] + ".*")
+        filename = glob.glob(os.path.join(datapath, f"{fdatetime}{datatype_list[0]}.*"))
     elif datatype_list[0] == "Nh":
-        filename = glob.glob(datapath + fdatetime + "dBZ.*")
+        filename = glob.glob(os.path.join(datapath, f"{fdatetime}dBZ.*"))
     else:
-        filename = glob.glob(datapath + fdatetime + "dBZv.*")
+        filename = glob.glob(os.path.join(datapath, f"{fdatetime}dBZv.*"))
 
     # create radar object
     radar = None
     if not filename:
-        warn("No file found in " + datapath + fdatetime + datatype_list[0] + ".*")
+        warn("No file found in " + os.path.join(datapath, f"{fdatetime}{datatype_list[0]}.*"))
     else:
         radar = get_data_rainbow(filename[0], datatype_list[0])
 
@@ -4104,13 +4103,14 @@ def merge_fields_rainbow(basepath, scan_name, voltime, datatype_list):
     # add other fields in the same scan
     for datatype in datatype_list[1:]:
         if datatype not in ("Nh", "Nv"):
-            filename = glob.glob(datapath + fdatetime + datatype + ".*")
+            filename = glob.glob(os.path.join(datapath, f"{fdatetime}{datatype}.*"))
         elif datatype == "Nh":
-            filename = glob.glob(datapath + fdatetime + "dBZ.*")
+            filename = glob.glob(os.path.join(datapath, f"{fdatetime}dBZ.*"))
         else:
-            filename = glob.glob(datapath + fdatetime + "dBZv.*")
+            filename = glob.glob(os.path.join(datapath, f"{fdatetime}dBZv.*"))
+
         if not filename:
-            warn("No file found in " + datapath + fdatetime + datatype + ".*")
+            warn("No file found in " + os.path.join(datapath, f"{fdatetime}{datatype}.*"))
         else:
             radar_aux = get_data_rainbow(filename[0], datatype)
             if radar_aux is None:
@@ -4132,6 +4132,7 @@ def merge_fields_rainbow(basepath, scan_name, voltime, datatype_list):
     return radar
 
 
+
 def merge_fields_psr_spectra(
     basepath,
     basepath_psr,

src/pyrad_proc/pyrad/io/read_data_sensor.py

@@ -2226,7 +2226,7 @@ def get_sensor_data(date, datatype, cfg):
                                 + "_" + cfg["sensorid"] + ".csv*")
         datafile2 = os.path.join(datapath2, date.strftime("%Y%m%d")
                                 + "_" + cfg["sensorid"] + ".csv*")
-        
+
         files1 = glob.glob(datafile1)
         files2 = glob.glob(datafile2)
         if len(files1):

src/pyrad_proc/pyrad/prod/process_vol_products.py

@@ -3690,7 +3690,6 @@ def generate_vol_products(dataset, prdcfg):
         field_data_process = field_data_process[:,ind_rng]
 
         countzero = np.size(np.where(field_data_process == 0.0))
-
         meanval = np.nanmean(field_data_process)
         if write_min_max:
             minval = np.nanmin(field_data_process)

tools/processData/launch_sbatch.sbatch

@@ -1,24 +1,56 @@
-#!/bin/bash -l
-#SBATCH --time=24:00:00
-#SBATCH --nodes=1
-#SBATCH --ntasks=1
-#SBATCH --cpus-per-task=4
-#SBATCH --ntasks-per-node=1
-#SBATCH --mem=64g
-#SBATCH --partition=postproc
-#SBATCH --account=msrad
+#!/bin/bash
+#SBATCH -N 1                 # nodes requested
+#SBATCH -c 1                 # cores requested
+#SBATCH -t 10:0:00           # time requested in hour:minute:second
+#SBATCH --partition postproc
+
+# Default values
+DEFAULT_PYART_CONFIG="$HOME/pyrad/config/pyart/mch_config.py"
+DEFAULT_CONDA_ENV_NAME="pyart"
+
+# Assign arguments or defaults
+PYART_CONFIG=${3:-$DEFAULT_PYART_CONFIG}
+
+export PYTHONUNBUFFERED=1
+export PATH=$PATH
+export PYART_CONFIG=$PYART_CONFIG
+export CONDA_ENV_NAME=$DEFAULT_CONDA_ENV_NAME
+
+current_time=$(date +'%Y%m%d%H%M%S')
+
+echo $current_time
+echo "Launched executable $EXECUTABLE"
+
+source activate $CONDA_ENV_NAME
+
+$EXECUTABLE
+
+source deactivate
+
+
+
+#!/bin/bash
+#SBATCH -N 1     # nodes requested
+#SBATCH -c 1      # cores requested
+#SBATCH -t 10:0:00  # time requested in hour:minute:second
+#SBATCH --partition postproc
 
 # other sbatch options:
 # mem-per-cpu : maximum amount of real memory per allocated cpu required by the job
 #               --mem >= --mem-per-cpu if --mem is specified
 
 EXECUTABLE=$1
 
+export PYTHONUNBUFFERED=1
+export PATH=/store_new/mch/msrad/utils/miniconda-wolfensb/envs/pyart/bin/:$PATH
+export PYART_CONFIG=$HOME/pyrad/config/pyart/mch_config.py
+current_time=$(date +'%Y%m%d%H%M%S')
+
+echo
 echo "Launched executable $EXECUTABLE"
 
-source activate pyrad
+source activate pyart
 
 $EXECUTABLE
 
-
 source deactivate