From 4be061792287fe10b801a8b3dcb5f8148cb5ada9 Mon Sep 17 00:00:00 2001 From: Greg Starr Date: Sat, 25 Jun 2022 12:21:16 -0400 Subject: [PATCH 1/5] hopfully improving madrigal download robustness --- trough/_download.py | 52 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 18 deletions(-) diff --git a/trough/_download.py b/trough/_download.py index 1da1953..6dbd835 100644 --- a/trough/_download.py +++ b/trough/_download.py @@ -11,6 +11,7 @@ import functools import logging import warnings + try: import h5py from madrigalWeb import madrigalWeb @@ -18,10 +19,10 @@ except ImportError as imp_err: warnings.warn(f"Packages required for recreating dataset not installed: {imp_err}") - from trough.exceptions import InvalidConfiguration from trough._arb import parse_arb_fn +RETRIES = 3 logger = logging.getLogger(__name__) @@ -134,16 +135,22 @@ def __init__(self, download_dir, user_name, user_email, user_affil): def _get_tec_experiments(self, start_date: datetime, end_date: datetime): logger.info(f"getting TEC experiments between {start_date} and {end_date}") - experiments = self.server.getExperiments( - 8000, - start_date.year, start_date.month, start_date.day, start_date.hour, start_date.minute, start_date.second, - end_date.year, end_date.month, end_date.day, end_date.hour, end_date.minute, end_date.second, - ) - return experiments - - def _download_file(self, tec_file, local_path, retries=3): + for retry in range(RETRIES): + try: + return self.server.getExperiments( + 8000, + start_date.year, start_date.month, start_date.day, start_date.hour, start_date.minute, + start_date.second, end_date.year, end_date.month, end_date.day, end_date.hour, end_date.minute, + end_date.second, + ) + except ValueError as e: + logger.warning(f'Failure getting experiments, retrying {retry}') + self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") + raise e + + def _download_file(self, tec_file, local_path): logger.info(f"downloading TEC file {tec_file} to {local_path}") - for retry in range(retries): + for retry in range(RETRIES): try: if pathlib.Path(local_path).exists(): logger.info(f"already exists: {local_path}") @@ -151,9 +158,10 @@ def _download_file(self, tec_file, local_path, retries=3): return self.server.downloadFile( tec_file, local_path, self.user_name, self.user_email, self.user_affil, 'hdf5' ) - except(socket.timeout, TimeoutError): - logger.error(f'Failure downloading {tec_file} because it took more than allowed number of seconds') + except(socket.timeout, TimeoutError) as e: + logger.warning(f'Failure downloading {tec_file}') self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") + raise e def _download_files(self, files): local_files = [] @@ -178,8 +186,16 @@ def _get_file_list(self, start_date, end_date): if cache_key in self.cache: files = self.cache[cache_key] else: - experiment_files = self.server.getExperimentFiles(experiment.id) - files = [exp.name for exp in experiment_files if exp.kindat == 3500] + for retry in range(RETRIES): + try: + experiment_files = self.server.getExperimentFiles(experiment.id) + files = [exp.name for exp in experiment_files if exp.kindat == 3500] + break + except ValueError as e: + logger.warning(f'Failure getting experiment {experiment.id}') + self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") + else: + raise e tec_files[cache_key] = files return tec_files @@ -329,9 +345,9 @@ def _verify_files(self, local_files, server_files): return bad_server_files -def _download_ftp_file(server, server_file: str, local_path: str, retries=3): +def _download_ftp_file(server, server_file: str, local_path: str): logger.info(f"downloading file {server_file} to {local_path}") - for retry in range(retries): + for retry in range(RETRIES): try: with open(local_path, 'wb') as f: server.retrbinary(f'RETR {str(server_file)}', f.write) @@ -340,9 +356,9 @@ def _download_ftp_file(server, server_file: str, local_path: str, retries=3): logger.error(f'Failure downloading {server_file} because it took more than allowed number of seconds') -def _download_http_file(http_file: str, local_path: str, retries=3): +def _download_http_file(http_file: str, local_path: str): logger.info(f"downloading file {http_file} to {local_path}") - for retry in range(retries): + for retry in range(RETRIES): try: with request.urlopen(http_file, timeout=60) as r: with open(local_path, 'wb') as f: From df89e41c19c077a1bd41fa716c3ed69b596ef10a Mon Sep 17 00:00:00 2001 From: Greg Starr Date: Sat, 25 Jun 2022 13:08:14 -0400 Subject: [PATCH 2/5] making madrigal download more robust and fixing flake8 issues --- trough/_download.py | 31 +++++++++++++++++++------------ trough/_trough.py | 6 +++++- trough/utils.py | 2 +- 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/trough/_download.py b/trough/_download.py index 6dbd835..f01c410 100644 --- a/trough/_download.py +++ b/trough/_download.py @@ -135,6 +135,7 @@ def __init__(self, download_dir, user_name, user_email, user_affil): def _get_tec_experiments(self, start_date: datetime, end_date: datetime): logger.info(f"getting TEC experiments between {start_date} and {end_date}") + err = Exception for retry in range(RETRIES): try: return self.server.getExperiments( @@ -143,13 +144,14 @@ def _get_tec_experiments(self, start_date: datetime, end_date: datetime): start_date.second, end_date.year, end_date.month, end_date.day, end_date.hour, end_date.minute, end_date.second, ) - except ValueError as e: + except ValueError as err: logger.warning(f'Failure getting experiments, retrying {retry}') self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") - raise e + raise err def _download_file(self, tec_file, local_path): logger.info(f"downloading TEC file {tec_file} to {local_path}") + err = Exception for retry in range(RETRIES): try: if pathlib.Path(local_path).exists(): @@ -158,10 +160,10 @@ def _download_file(self, tec_file, local_path): return self.server.downloadFile( tec_file, local_path, self.user_name, self.user_email, self.user_affil, 'hdf5' ) - except(socket.timeout, TimeoutError) as e: + except(socket.timeout, TimeoutError) as err: logger.warning(f'Failure downloading {tec_file}') self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") - raise e + raise err def _download_files(self, files): local_files = [] @@ -186,16 +188,17 @@ def _get_file_list(self, start_date, end_date): if cache_key in self.cache: files = self.cache[cache_key] else: + err = Exception for retry in range(RETRIES): try: experiment_files = self.server.getExperimentFiles(experiment.id) files = [exp.name for exp in experiment_files if exp.kindat == 3500] break - except ValueError as e: + except ValueError as err: logger.warning(f'Failure getting experiment {experiment.id}') self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") else: - raise e + raise err tec_files[cache_key] = files return tec_files @@ -205,8 +208,8 @@ def _verify_local_file(local_file): with h5py.File(local_file, 'r') as f: tec = f['Data']['Array Layout']['2D Parameters']['tec'][()] timestamps = f['Data']['Array Layout']['timestamps'][()] - except Exception as e: - logger.warning(f"bad local file: {local_file}, error: {e}") + except Exception as err: + logger.warning(f"bad local file: {local_file}, error: {err}") return False return (timestamps.shape[0] > 10) and (np.sum(np.isfinite(tec)) > 100) @@ -332,8 +335,8 @@ def _verify_local_file(local_file): try: with h5py.File(local_file, 'r') as f: lon = f['MODEL_NORTH_GEOGRAPHIC_LONGITUDE'][()] - except Exception as e: - logger.warning(f"bad local file: {local_file}, error: {e}") + except Exception as err: + logger.warning(f"bad local file: {local_file}, error: {err}") return False return lon.shape[0] > 10 @@ -347,22 +350,26 @@ def _verify_files(self, local_files, server_files): def _download_ftp_file(server, server_file: str, local_path: str): logger.info(f"downloading file {server_file} to {local_path}") + err = Exception for retry in range(RETRIES): try: with open(local_path, 'wb') as f: server.retrbinary(f'RETR {str(server_file)}', f.write) return - except(socket.timeout, TimeoutError): + except(socket.timeout, TimeoutError) as err: logger.error(f'Failure downloading {server_file} because it took more than allowed number of seconds') + raise err def _download_http_file(http_file: str, local_path: str): logger.info(f"downloading file {http_file} to {local_path}") + err = Exception for retry in range(RETRIES): try: with request.urlopen(http_file, timeout=60) as r: with open(local_path, 'wb') as f: f.write(r.read()) return - except(socket.timeout, TimeoutError): + except(socket.timeout, TimeoutError) as err: logger.error(f'Failure downloading {http_file} because it took more than allowed number of seconds') + raise err diff --git a/trough/_trough.py b/trough/_trough.py index f83f987..a0eb610 100644 --- a/trough/_trough.py +++ b/trough/_trough.py @@ -79,7 +79,11 @@ def _get_weighted_kp(times, omni_data, tau=.6, T=10): values = 2.1 * np.log(.2 * ap_tau + 1) if (times.values[1] - times.values[0]).astype('timedelta64[m]').astype(int) == 60: return values - ut_initial = np.arange(times.values[0], times.values[-1] + np.timedelta64(1, 'h'), np.timedelta64(1, 'h')).astype('datetime64[s]').astype(int) + ut_initial = np.arange( + times.values[0], + times.values[-1] + np.timedelta64(1, 'h'), + np.timedelta64(1, 'h') + ).astype('datetime64[s]').astype(int) ut_final = times.values.astype('datetime64[s]').astype(int) return np.interp(ut_final, ut_initial, values) diff --git a/trough/utils.py b/trough/utils.py index e43a035..6f5eb34 100644 --- a/trough/utils.py +++ b/trough/utils.py @@ -147,7 +147,7 @@ def check(start, end, dt, hemisphere, processed_file): logger.info(f"downloaded data already processed {processed_file=}, checking...") return False except KeyError: - logger.info(f"processed file doesn't have the requested data") + logger.info("processed file doesn't have the requested data") except Exception as e: logger.info(f"error reading processed file {processed_file=}: {e}, removing and reprocessing") processed_file.unlink() From c1d2e883064ed7b56ca439c0478ca8464c54cd83 Mon Sep 17 00:00:00 2001 From: Greg Starr Date: Sat, 25 Jun 2022 14:47:40 -0400 Subject: [PATCH 3/5] asdfasfdafsddf --- trough/_download.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/trough/_download.py b/trough/_download.py index f01c410..e6a3923 100644 --- a/trough/_download.py +++ b/trough/_download.py @@ -1,3 +1,4 @@ +import time import numpy as np from datetime import datetime, timedelta import math @@ -146,7 +147,7 @@ def _get_tec_experiments(self, start_date: datetime, end_date: datetime): ) except ValueError as err: logger.warning(f'Failure getting experiments, retrying {retry}') - self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") + time.sleep(10) raise err def _download_file(self, tec_file, local_path): @@ -162,7 +163,7 @@ def _download_file(self, tec_file, local_path): ) except(socket.timeout, TimeoutError) as err: logger.warning(f'Failure downloading {tec_file}') - self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") + time.sleep(10) raise err def _download_files(self, files): @@ -196,7 +197,7 @@ def _get_file_list(self, start_date, end_date): break except ValueError as err: logger.warning(f'Failure getting experiment {experiment.id}') - self.server = madrigalWeb.MadrigalData("http://cedar.openmadrigal.org") + time.sleep(10) else: raise err tec_files[cache_key] = files @@ -358,6 +359,7 @@ def _download_ftp_file(server, server_file: str, local_path: str): return except(socket.timeout, TimeoutError) as err: logger.error(f'Failure downloading {server_file} because it took more than allowed number of seconds') + time.sleep(10) raise err @@ -372,4 +374,5 @@ def _download_http_file(http_file: str, local_path: str): return except(socket.timeout, TimeoutError) as err: logger.error(f'Failure downloading {http_file} because it took more than allowed number of seconds') + time.sleep(10) raise err From 9e744bbd7646d73c736f2a9607f34a1a835fa090 Mon Sep 17 00:00:00 2001 From: Greg Starr Date: Sun, 26 Jun 2022 00:27:49 -0400 Subject: [PATCH 4/5] fixing test robustness issue --- trough/_download.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trough/_download.py b/trough/_download.py index e6a3923..2cc1550 100644 --- a/trough/_download.py +++ b/trough/_download.py @@ -161,7 +161,7 @@ def _download_file(self, tec_file, local_path): return self.server.downloadFile( tec_file, local_path, self.user_name, self.user_email, self.user_affil, 'hdf5' ) - except(socket.timeout, TimeoutError) as err: + except ValueError as err: logger.warning(f'Failure downloading {tec_file}') time.sleep(10) raise err From 917e8b371f9c9029dd53138f197539e7a4e76d28 Mon Sep 17 00:00:00 2001 From: Greg Starr Date: Mon, 27 Jun 2022 23:18:17 -0400 Subject: [PATCH 5/5] actually fixing download robustness issue this time I think --- trough/_download.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/trough/_download.py b/trough/_download.py index 2cc1550..2cefd78 100644 --- a/trough/_download.py +++ b/trough/_download.py @@ -152,19 +152,19 @@ def _get_tec_experiments(self, start_date: datetime, end_date: datetime): def _download_file(self, tec_file, local_path): logger.info(f"downloading TEC file {tec_file} to {local_path}") - err = Exception - for retry in range(RETRIES): - try: - if pathlib.Path(local_path).exists(): - logger.info(f"already exists: {local_path}") - else: + if pathlib.Path(local_path).exists(): + logger.info(f"already exists: {local_path}") + else: + err = Exception + for retry in range(RETRIES): + try: return self.server.downloadFile( tec_file, local_path, self.user_name, self.user_email, self.user_affil, 'hdf5' ) - except ValueError as err: - logger.warning(f'Failure downloading {tec_file}') - time.sleep(10) - raise err + except ValueError as err: + logger.warning(f'Failure downloading {tec_file}') + time.sleep(10) + raise err def _download_files(self, files): local_files = []