From ccf191fa3f4f74cfa1975ef50360465318b9a5aa Mon Sep 17 00:00:00 2001 From: Michael Thornton Date: Tue, 10 Dec 2024 14:26:29 -0800 Subject: [PATCH] delete large objects and data structs after use --- nmdc_automation/run_process/run_import.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/nmdc_automation/run_process/run_import.py b/nmdc_automation/run_process/run_import.py index 970f3791..6301b3aa 100644 --- a/nmdc_automation/run_process/run_import.py +++ b/nmdc_automation/run_process/run_import.py @@ -1,5 +1,6 @@ import click import csv +import gc import importlib.resources from functools import lru_cache import logging @@ -70,6 +71,8 @@ def import_projects(import_file, import_yaml, site_configuration, iteration): # validate the database logger.info("Validating imported data") db_dict = yaml.safe_load(yaml_dumper.dumps(db)) + del db # free up memory + del do_mapping # free up memory validation_report = linkml.validator.validate(db_dict, nmdc_materialized) if validation_report.results: logger.error(f"Validation Failed") @@ -93,10 +96,13 @@ def import_projects(import_file, import_yaml, site_configuration, iteration): logger.info("Posting data to the API") try: runtime.post_objects(db_dict) + del db_dict # free up memory except Exception as e: logger.error(f"Error posting data to the API: {e}") raise e + gc.collect() +