diff --git a/nmdc_automation/run_process/run_import.py b/nmdc_automation/run_process/run_import.py index 970f3791..6301b3aa 100644 --- a/nmdc_automation/run_process/run_import.py +++ b/nmdc_automation/run_process/run_import.py @@ -1,5 +1,6 @@ import click import csv +import gc import importlib.resources from functools import lru_cache import logging @@ -70,6 +71,8 @@ def import_projects(import_file, import_yaml, site_configuration, iteration): # validate the database logger.info("Validating imported data") db_dict = yaml.safe_load(yaml_dumper.dumps(db)) + del db # free up memory + del do_mapping # free up memory validation_report = linkml.validator.validate(db_dict, nmdc_materialized) if validation_report.results: logger.error(f"Validation Failed") @@ -93,10 +96,13 @@ def import_projects(import_file, import_yaml, site_configuration, iteration): logger.info("Posting data to the API") try: runtime.post_objects(db_dict) + del db_dict # free up memory except Exception as e: logger.error(f"Error posting data to the API: {e}") raise e + gc.collect() +