From 44d1bf7fb20fc5e1c9b566bca7809b43da30824c Mon Sep 17 00:00:00 2001 From: Joel Klinger Date: Mon, 2 Dec 2024 13:32:49 +0000 Subject: [PATCH 1/2] [feature/PI-618-bulk_etl] Layer updates --- .pre-commit-config.yaml | 2 +- src/layers/domain/api/sds/query.py | 59 +++-- .../domain/api/tests/test_search_query.py | 40 ---- src/layers/domain/core/device/v1.py | 4 - src/layers/domain/core/validation.py | 2 +- .../spine_as/field_mapping.json | 2 +- src/layers/etl_utils/ldif/_ldif.py | 2 +- src/layers/etl_utils/ldif/ldif.py | 13 +- src/layers/etl_utils/ldif/tests/test_ldif.py | 67 ++++++ src/layers/etl_utils/worker/exception.py | 14 +- .../sds/domain/nhs_accredited_system.py | 2 +- src/layers/sds/epr/bulk_create/bulk_create.py | 67 ++++-- .../sds/epr/bulk_create/bulk_load_fanout.py | 21 ++ .../sds/epr/bulk_create/bulk_repository.py | 33 ++- src/layers/sds/epr/bulk_create/getters.py | 24 +- .../sds/epr/bulk_create/tests/test_getters.py | 221 +----------------- src/layers/sds/epr/tags/tags.py | 4 +- 17 files changed, 220 insertions(+), 357 deletions(-) create mode 100644 src/layers/sds/epr/bulk_create/bulk_load_fanout.py diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6dc0eb8cd..4c80c758d 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,7 +4,7 @@ repos: rev: v1.4.0 hooks: - id: detect-secrets - exclude: ".pre-commit-config.yaml|infrastructure/localstack/provider.tf|src/etl/sds/tests/changelog" + exclude: ".pre-commit-config.yaml|infrastructure/localstack/provider.tf|src/etl/sds/tests/changelog|src/etl/sds/worker/bulk/transform_bulk/tests|src/etl/sds/worker/bulk/tests/stage_data" - repo: https://github.com/prettier/pre-commit rev: 57f39166b5a5a504d6808b87ab98d41ebf095b46 diff --git a/src/layers/domain/api/sds/query.py b/src/layers/domain/api/sds/query.py index cece33e11..6dc3fb035 100644 --- a/src/layers/domain/api/sds/query.py +++ b/src/layers/domain/api/sds/query.py @@ -1,49 +1,39 @@ -from functools import cache -from itertools import chain, combinations - from pydantic import BaseModel, Extra, root_validator -class SearchSDSQueryParams(BaseModel): +class SearchSDSDeviceQueryParams(BaseModel, extra=Extra.forbid): + nhs_id_code: str + nhs_as_svc_ia: str + nhs_mhs_manufacturer_org: str = None + nhs_mhs_party_key: str = None + + @root_validator + def client_to_id(cls, values): + nhs_as_client = values.get("nhs_as_client") + nhs_id_code = values.get("nhs_id_code") + if nhs_as_client and not nhs_id_code: + values["nhs_id_code"] = nhs_as_client + return values + def get_non_null_params(self): return self.dict(exclude_none=True) @classmethod - @cache def allowed_field_combinations(cls) -> list[set[str]]: - """ - This method is used to generate all allowed combinations of search fields - for the given query parameters. Down the line this also used to generate - Device.tags in the ETL - """ - mandatory_fields, optional_fields = [], [] - for field_name, field in cls.__fields__.items(): - if field.required: - mandatory_fields.append(field_name) - else: - optional_fields.append(field_name) - - n_minimum_optional_fields = 0 if mandatory_fields else 1 - n_optional_fields = len(optional_fields) - optional_field_combinations = chain.from_iterable( - combinations(optional_fields, n_fields) - for n_fields in range(n_minimum_optional_fields, n_optional_fields + 1) - ) - return [ - {*mandatory_fields, *_optional_field_combination} - for _optional_field_combination in optional_field_combinations + {"nhs_id_code", "nhs_as_svc_ia"}, + {"nhs_id_code", "nhs_as_svc_ia", "nhs_mhs_party_key"}, ] -class SearchSDSDeviceQueryParams(SearchSDSQueryParams, extra=Extra.forbid): - nhs_as_client: str - nhs_as_svc_ia: str - nhs_mhs_manufacturer_org: str = None - nhs_mhs_party_key: str = None - +class SearchSDSEndpointQueryParams(BaseModel, extra=Extra.forbid): + # can effectively achieve these without tags now + # if + # 1. nhs_id_code and nhs_mhs_party_key -> only need party key + # 2. nhs_id_code and nhs_mhs_svc_ia -> can retrieve all devices for org, and filter on nhs_mhs_svc_ia + # 3. nhs_mhs_party_key and nhs_mhs_svc_ia -> can retrieve the mhs device for product, and filter on nhs_mhs_svc_ia --> possible that can read device directly by cpaid + # 4. nhs_id_code and nhs_mhs_party_key and nhs_mhs_svc_ia -> same as 3. -class SearchSDSEndpointQueryParams(SearchSDSQueryParams, extra=Extra.forbid): nhs_id_code: str = None nhs_mhs_svc_ia: str = None nhs_mhs_party_key: str = None @@ -59,3 +49,6 @@ def check_filters(cls, values: dict): "At least 2 query parameters should be provided of type, nhs_id_code, nhs_mhs_svc_ia and nhs_mhs_party_key" ) return values + + def get_non_null_params(self): + return self.dict(exclude_none=True) diff --git a/src/layers/domain/api/tests/test_search_query.py b/src/layers/domain/api/tests/test_search_query.py index 4be8358b6..7809fb7e2 100644 --- a/src/layers/domain/api/tests/test_search_query.py +++ b/src/layers/domain/api/tests/test_search_query.py @@ -1,10 +1,7 @@ -from typing import Optional - import pytest from domain.api.sds.query import ( SearchSDSDeviceQueryParams, SearchSDSEndpointQueryParams, - SearchSDSQueryParams, ) from pydantic import ValidationError @@ -97,40 +94,3 @@ def test_endpoint_query_accepted(params): def test_endpoint_query_invalid(params): with pytest.raises(ValidationError): search = SearchSDSEndpointQueryParams(**params) - - -def test_allowed_field_combinations(): - class MyModel(SearchSDSQueryParams): - foo: str - bar: Optional[str] - bob: Optional[str] - - assert MyModel.allowed_field_combinations() == [ - {"foo"}, - {"foo", "bar"}, - {"foo", "bob"}, - {"foo", "bar", "bob"}, - ] - - -def test_allowed_field_combinations_all_optional(): - class MyModel(SearchSDSQueryParams): - foo: Optional[str] - bar: Optional[str] - bob: Optional[str] - - assert MyModel.allowed_field_combinations() == [ - { - "foo", - }, - { - "bar", - }, - { - "bob", - }, - {"foo", "bar"}, - {"foo", "bob"}, - {"bar", "bob"}, - {"foo", "bar", "bob"}, - ] diff --git a/src/layers/domain/core/device/v1.py b/src/layers/domain/core/device/v1.py index 9036a60d8..d38c5a52c 100644 --- a/src/layers/domain/core/device/v1.py +++ b/src/layers/domain/core/device/v1.py @@ -159,10 +159,6 @@ class DeviceTagsClearedEvent(Event): @dataclass(kw_only=True, slots=True) class QuestionnaireResponseUpdatedEvent(Event): - """ - This is adding the initial questionnaire response from the event body request. - """ - id: str questionnaire_responses: dict[str, list[QuestionnaireResponse]] keys: list[DeviceKey] diff --git a/src/layers/domain/core/validation.py b/src/layers/domain/core/validation.py index cde59125d..8c4deec9b 100644 --- a/src/layers/domain/core/validation.py +++ b/src/layers/domain/core/validation.py @@ -31,7 +31,7 @@ class AccreditedSystem: ID_PATTERN = re.compile(rf"^[a-zA-Z-0-9]+$") class PartyKey: - PARTY_KEY_REGEX = rf"^{_ODS_CODE_REGEX}-[0-9]{{6}}$" + PARTY_KEY_REGEX = rf"^{_ODS_CODE_REGEX}-[0-9]{{6,9}}$" ID_PATTERN = re.compile(PARTY_KEY_REGEX) class CpaId: diff --git a/src/layers/domain/repository/questionnaire_repository/v1/questionnaires/spine_as/field_mapping.json b/src/layers/domain/repository/questionnaire_repository/v1/questionnaires/spine_as/field_mapping.json index 7ae8ed83c..cfe778e9d 100644 --- a/src/layers/domain/repository/questionnaire_repository/v1/questionnaires/spine_as/field_mapping.json +++ b/src/layers/domain/repository/questionnaire_repository/v1/questionnaires/spine_as/field_mapping.json @@ -14,5 +14,5 @@ "nhs_as_acf": "AS ACF", "nhs_temp_uid": "Temp UID", "description": "Description", - "nhs_as_category_bag": "AS Category Bag" + "nhs_as_category_bag": "Category Bag" } diff --git a/src/layers/etl_utils/ldif/_ldif.py b/src/layers/etl_utils/ldif/_ldif.py index 5abde19be..22bc33f7d 100644 --- a/src/layers/etl_utils/ldif/_ldif.py +++ b/src/layers/etl_utils/ldif/_ldif.py @@ -203,7 +203,7 @@ def __next_key_and_value(self): # All values should be valid ascii; we support UTF-8 as a # non-official, backwards compatibility layer. attr_value = unfolded_line[colon_pos + 1 :].encode("utf-8") - return attr_type.lower(), attr_value + return attr_type.lower(), attr_value.strip() def _consume_empty_lines(self): """ diff --git a/src/layers/etl_utils/ldif/ldif.py b/src/layers/etl_utils/ldif/ldif.py index d5b53c204..103b16bc6 100644 --- a/src/layers/etl_utils/ldif/ldif.py +++ b/src/layers/etl_utils/ldif/ldif.py @@ -1,4 +1,5 @@ import re +from base64 import b64decode from collections import defaultdict from io import BytesIO from types import FunctionType @@ -109,11 +110,16 @@ def __init__(self, group_field: str, filter_terms: list[tuple[str, str]]): for key, value in filter_terms ] self.group_filter = re.compile(rf"(?i)^({group_field}): (.*)\n$".encode()).match + self.group_filter_base_64 = re.compile( + rf"(?i)^({group_field}):: (.*)\n$".encode() + ).match self.reset() def flush(self) -> str: if self.group is None: - raise Exception + raise ValueError( + f"No group name assigned to the following group:\n{self.buffer}" + ) self.data[self.group].write(self.buffer) self.reset() @@ -126,6 +132,11 @@ def parse(self, line: bytes): group_match = self.group_filter(line) if group_match: (_, self.group) = group_match.groups() + else: + b64_group_match = self.group_filter_base_64(line) + if b64_group_match: + (_, b64_group) = b64_group_match.groups() + self.group = b64decode(b64_group).strip() if not self.keep and any(filter(line) for filter in self.filters): self.keep = True diff --git a/src/layers/etl_utils/ldif/tests/test_ldif.py b/src/layers/etl_utils/ldif/tests/test_ldif.py index d67bc2db5..758bf8fc5 100644 --- a/src/layers/etl_utils/ldif/tests/test_ldif.py +++ b/src/layers/etl_utils/ldif/tests/test_ldif.py @@ -296,6 +296,54 @@ myOtherField: 123 """ +LDIF_TO_FILTER_AND_GROUP_EXAMPLE_BASE_64 = """ +dn: uniqueIdentifier=AAA1 +myField:: QUFB +myOtherField: 123 + +dn: uniqueIdentifier=BBB1 +myfield: BBB +myOtherField: 123 + +dn: uniqueIdentifier=BBB2 +myfield: BBB +myOtherField: 123 + +dn: uniqueIdentifier=AAA2 +myfield: AAA +myOtherField: 123 + +dn: uniqueIdentifier=AAA3 +myField: AAA +myOtherField: 234 + +dn: uniqueIdentifier=BBB3 +myfield:: QkJC +myOtherField: 123 +""" + +FILTERED_AND_GROUPED_LDIF_TO_FILTER_AND_GROUP_EXAMPLE = """ +dn: uniqueIdentifier=AAA1 +myField:: QUFB +myOtherField: 123 + +dn: uniqueIdentifier=AAA2 +myfield: AAA +myOtherField: 123 + +dn: uniqueIdentifier=BBB1 +myfield: BBB +myOtherField: 123 + +dn: uniqueIdentifier=BBB2 +myfield: BBB +myOtherField: 123 + +dn: uniqueIdentifier=BBB3 +myfield:: QkJC +myOtherField: 123 +""" + @pytest.mark.parametrize( ("raw_distinguished_name", "parsed_distinguished_name"), @@ -390,6 +438,25 @@ def test_filter_and_group_ldif_from_s3_by_property(mocked_open): ) +@mock.patch( + "etl_utils.ldif.ldif._smart_open", + return_value=BytesIO(LDIF_TO_FILTER_AND_GROUP_EXAMPLE_BASE_64.encode()), +) +def test_filter_and_group_ldif_from_s3_by_property_with_b64encoded_group(mocked_open): + with mock_aws(): + s3_client = boto3.client("s3") + filtered_ldif = filter_and_group_ldif_from_s3_by_property( + s3_client=s3_client, + s3_path="s3://dummy_bucket/dummy_key", + group_field="myField", + filter_terms=[("myOtherField", "123")], + ) + assert ( + "".join(data.tobytes().decode() for data in filtered_ldif) + == FILTERED_AND_GROUPED_LDIF_TO_FILTER_AND_GROUP_EXAMPLE + ) + + @pytest.mark.parametrize( ["raw_ldif", "parsed_ldif"], [ diff --git a/src/layers/etl_utils/worker/exception.py b/src/layers/etl_utils/worker/exception.py index ca11d6d73..86ce535d6 100644 --- a/src/layers/etl_utils/worker/exception.py +++ b/src/layers/etl_utils/worker/exception.py @@ -25,6 +25,14 @@ TRUNCATED = "[TRUNCATED]\n" +def truncate_message( + item: str, truncation_depth=TRUNCATION_DEPTH, truncated_note=TRUNCATED +): + if len(item) > truncation_depth: + return item[:truncation_depth] + truncated_note + return item + + def _render_exception(exception: Exception) -> str: """Concatenates an exception with its notes""" _notes = exception.__dict__.get("__notes__", []) @@ -92,9 +100,11 @@ def render_exception( _traceback = "".join(TracebackException.from_exception(exception).format()) if truncation_depth is not None and len(formatted_exception) > truncation_depth: - formatted_exception = formatted_exception[:truncation_depth] + TRUNCATED + formatted_exception = truncate_message( + formatted_exception, truncation_depth=truncation_depth + ) if truncation_depth is not None and len(_traceback) > truncation_depth: - _traceback = _traceback[:truncation_depth] + TRUNCATED + _traceback = truncate_message(_traceback, truncation_depth=truncation_depth) return f"{indentation}{formatted_exception}\n{_traceback}\n" diff --git a/src/layers/sds/domain/nhs_accredited_system.py b/src/layers/sds/domain/nhs_accredited_system.py index b92b2d322..3e63ab19b 100644 --- a/src/layers/sds/domain/nhs_accredited_system.py +++ b/src/layers/sds/domain/nhs_accredited_system.py @@ -27,7 +27,7 @@ class NhsAccreditedSystem(SdsBaseModel): nhs_product_name: Optional[str] = Field(alias="nhsproductname") nhs_product_version: Optional[str] = Field(alias="nhsproductversion") nhs_as_acf: Optional[set[str]] = Field(alias="nhsasacf") - nhs_as_client: Optional[set[str]] = Field(alias="nhsasclient") + nhs_as_client: Optional[set[str]] = Field(alias="nhsasclient", default_factory=set) nhs_as_svc_ia: set[str] = Field(alias="nhsassvcia") nhs_temp_uid: Optional[str] = Field(alias="nhstempuid") description: Optional[str] = Field(alias="description") diff --git a/src/layers/sds/epr/bulk_create/bulk_create.py b/src/layers/sds/epr/bulk_create/bulk_create.py index 53334bc7a..06e17fd45 100644 --- a/src/layers/sds/epr/bulk_create/bulk_create.py +++ b/src/layers/sds/epr/bulk_create/bulk_create.py @@ -31,7 +31,7 @@ def _create_complete_epr_product( mhs_tags: list[dict], additional_interactions_data: list[QuestionnaireResponse], as_device_data: list[QuestionnaireResponse], - as_tags: list[dict], + as_tags: list[list[dict]], product_team_ids: dict[str, str], ) -> list[AggregateRoot]: if ods_code in product_team_ids: @@ -50,14 +50,19 @@ def _create_complete_epr_product( message_sets = create_message_sets( product=product, party_key=party_key, message_set_data=message_set_data ) - mhs_device = create_mhs_device( - product=product, - party_key=party_key, - mhs_device_data=mhs_device_data, - cpa_ids=mhs_cpa_ids, - message_sets_id=message_sets.id, - mhs_tags=mhs_tags, - ) + + mhs_devices = [] + if mhs_device_data: + mhs_device = create_mhs_device( + product=product, + party_key=party_key, + mhs_device_data=mhs_device_data, + cpa_ids=mhs_cpa_ids, + message_sets_id=message_sets.id, + mhs_tags=mhs_tags, + ) + mhs_devices = [mhs_device] + additional_interactions = create_additional_interactions( product=product, party_key=party_key, @@ -71,9 +76,9 @@ def _create_complete_epr_product( as_device_data=data, additional_interactions_id=additional_interactions.id, message_sets_id=message_sets.id, - as_tags=as_tags, + as_tags=tags, ) - for data in as_device_data + for data, tags in zip(as_device_data, as_tags) ] return [ @@ -81,7 +86,7 @@ def _create_complete_epr_product( product, message_sets, additional_interactions, - mhs_device, + *mhs_devices, *as_devices, ] @@ -113,12 +118,25 @@ def create_complete_epr_product( # !! Assumption: All MHSs with this party key have the same basic metadata. # !! Could add a debug test here to ensure all of this metadata is equal # !! between MHSs with this party key. - first_mhs = message_handling_systems[0] - mhs_device_data = get_mhs_device_data( - mhs=first_mhs, - mhs_device_questionnaire=mhs_device_questionnaire, - mhs_device_field_mapping=mhs_device_field_mapping, - ) + ods_code, party_key, product_name = None, None, None + mhs_device_data = None + if message_handling_systems: + first_mhs = message_handling_systems[0] + mhs_device_data = get_mhs_device_data( + mhs=first_mhs, + mhs_device_questionnaire=mhs_device_questionnaire, + mhs_device_field_mapping=mhs_device_field_mapping, + ) + + ods_code = first_mhs["nhs_mhs_manufacturer_org"] + party_key = first_mhs["nhs_mhs_party_key"] + product_name = first_mhs["nhs_product_name"] or first_mhs["nhs_mhs_cpa_id"] + else: + first_as = accredited_systems[0] + ods_code = first_as["nhs_mhs_manufacturer_org"] + party_key = first_as["nhs_mhs_party_key"] + product_name = first_as["nhs_product_name"] + as_device_data = [ get_accredited_system_device_data( accredited_system=accredited_system, @@ -139,15 +157,18 @@ def create_complete_epr_product( ) mhs_tags = get_mhs_tags(message_handling_systems) - as_tags = get_accredited_system_tags(accredited_systems) + as_tags = [ + get_accredited_system_tags(accredited_system) + for accredited_system in accredited_systems + ] return _create_complete_epr_product( - ods_code=first_mhs["nhs_mhs_manufacturer_org"], - party_key=first_mhs["nhs_mhs_party_key"], - product_name=first_mhs["nhs_product_name"], + ods_code=ods_code, + party_key=party_key, + product_name=product_name, mhs_device_data=mhs_device_data, message_set_data=message_set_data, - mhs_cpa_ids=[mhs["nhs_mhs_cpa_id"] for mhs in message_handling_systems], + mhs_cpa_ids=[mhs["unique_identifier"] for mhs in message_handling_systems], mhs_tags=mhs_tags, additional_interactions_data=additional_interactions_data, as_device_data=as_device_data, diff --git a/src/layers/sds/epr/bulk_create/bulk_load_fanout.py b/src/layers/sds/epr/bulk_create/bulk_load_fanout.py new file mode 100644 index 000000000..a3b1ea137 --- /dev/null +++ b/src/layers/sds/epr/bulk_create/bulk_load_fanout.py @@ -0,0 +1,21 @@ +from math import ceil + +from domain.core.cpm_product.v1 import CpmProduct +from domain.core.device.v1 import Device +from domain.core.device_reference_data.v1 import DeviceReferenceData +from domain.core.product_team.v1 import ProductTeam + +FANOUT = 10 + + +def count_indexes(obj: Device | DeviceReferenceData | CpmProduct | ProductTeam): + count = 1 + if isinstance(obj, (Device, CpmProduct, ProductTeam)): + count += len(obj.keys) + if isinstance(obj, (Device)): + count += len(obj.tags) + return count + + +def calculate_batch_size(sequence: list, n_batches: int) -> int: + return ceil(len(sequence) / (n_batches)) or 1 diff --git a/src/layers/sds/epr/bulk_create/bulk_repository.py b/src/layers/sds/epr/bulk_create/bulk_repository.py index 8d9526af3..5c4f79432 100644 --- a/src/layers/sds/epr/bulk_create/bulk_repository.py +++ b/src/layers/sds/epr/bulk_create/bulk_repository.py @@ -1,7 +1,7 @@ import random import time from functools import wraps -from itertools import batched, chain +from itertools import batched from typing import TYPE_CHECKING from botocore.exceptions import ClientError @@ -151,28 +151,23 @@ def __init__(self, table_name, dynamodb_client, batch_size=10): table_name=None, dynamodb_client=None ) - def write(self, items: list[dict]): - def generate_transaction_statements(object_type_name, item): - handler_name = f"handle_{object_type_name}" - handler = getattr(self, handler_name) - batch_write_items = handler(item=item) - - if not isinstance(batch_write_items, list): - batch_write_items = [batch_write_items] - return batch_write_items - - batch_write_items = chain.from_iterable( - ( - generate_transaction_statements(object_type_name, item) - for _item in items - for (object_type_name, item) in _item.items() - ) - ) + def generate_transaction_statements( + self, item_with_name: dict[str, dict] + ) -> list[dict]: + ((object_type_name, item),) = item_with_name.items() + handler_name = f"handle_{object_type_name}" + handler = getattr(self, handler_name) + statements = handler(item=item) + if not isinstance(statements, list): + statements = [statements] + return statements + + def write(self, transaction_statements: list[dict]): responses = [ batch_write_chunk( client=self.client, table_name=self.table_name, chunk=chunk ) - for chunk in batched(batch_write_items, self.batch_size) + for chunk in batched(transaction_statements, self.batch_size) ] return responses diff --git a/src/layers/sds/epr/bulk_create/getters.py b/src/layers/sds/epr/bulk_create/getters.py index 35e596b6b..df4aa47d9 100644 --- a/src/layers/sds/epr/bulk_create/getters.py +++ b/src/layers/sds/epr/bulk_create/getters.py @@ -1,7 +1,4 @@ -from domain.api.sds.query import ( - SearchSDSDeviceQueryParams, - SearchSDSEndpointQueryParams, -) +from domain.api.sds.query import SearchSDSDeviceQueryParams from domain.core.questionnaire import Questionnaire, QuestionnaireResponse from sds.epr.constants import SdsFieldName from sds.epr.tags.tags import is_list_like, sds_metadata_to_device_tags @@ -52,12 +49,7 @@ def get_message_set_data( def get_mhs_tags(message_handling_systems: list[dict]) -> list[dict]: - tags = [] - for mhs in message_handling_systems: - tags += sds_metadata_to_device_tags( - data=mhs, model=SearchSDSEndpointQueryParams - ) - return [dict(tag) for tag in set(tags)] + return [] def get_additional_interactions_data( @@ -91,12 +83,8 @@ def get_accredited_system_device_data( ) -def get_accredited_system_tags( - accredited_systems: list[dict], -) -> list[dict]: - tags = [] - for accredited_system in accredited_systems: - tags += sds_metadata_to_device_tags( - data=accredited_system, model=SearchSDSDeviceQueryParams - ) +def get_accredited_system_tags(accredited_system: dict) -> list[dict]: + tags = sds_metadata_to_device_tags( + data=accredited_system, model=SearchSDSDeviceQueryParams + ) return [dict(tag) for tag in set(tags)] diff --git a/src/layers/sds/epr/bulk_create/tests/test_getters.py b/src/layers/sds/epr/bulk_create/tests/test_getters.py index 224dd5210..551dfaaaa 100644 --- a/src/layers/sds/epr/bulk_create/tests/test_getters.py +++ b/src/layers/sds/epr/bulk_create/tests/test_getters.py @@ -246,244 +246,45 @@ def test_get_mhs_tags(mhs_1: NhsMhs, mhs_2: NhsMhs): assert len(tags) == len(expected_tags) -def test_get_accredited_system_tags( - accredited_system_1: NhsAccreditedSystem, accredited_system_2: NhsAccreditedSystem -): - accredited_systems = [accredited_system_1.dict(), accredited_system_2.dict()] - tags = get_accredited_system_tags(accredited_systems=accredited_systems) +def test_get_accredited_system_tags(accredited_system_1: NhsAccreditedSystem): + tags = get_accredited_system_tags(accredited_system=accredited_system_1.dict()) expected_tags = [ { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", + "nhs_id_code": "AAA", "nhs_mhs_manufacturer_org": "AAA", "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", }, { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", + "nhs_id_code": "AAA", "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - }, - { - "nhs_as_client": "ABC", "nhs_as_svc_ia": "interaction-id-2", }, + {"nhs_id_code": "AAA", "nhs_as_svc_ia": "interaction-id-1"}, { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", + "nhs_id_code": "AAA", "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "CDE", "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-123456", }, { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", + "nhs_id_code": "AAA", "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", }, { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", + "nhs_id_code": "AAA", "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "EFG", "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_party_key": "AAA-456789", }, + {"nhs_id_code": "AAA", "nhs_as_svc_ia": "interaction-id-2"}, { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", + "nhs_id_code": "AAA", "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_as_client": "CDE", "nhs_as_svc_ia": "interaction-id-2", }, ] + for tag in tags: assert tag in expected_tags, f"{tag} not in expected tags" diff --git a/src/layers/sds/epr/tags/tags.py b/src/layers/sds/epr/tags/tags.py index fbd498915..5e97eed0c 100644 --- a/src/layers/sds/epr/tags/tags.py +++ b/src/layers/sds/epr/tags/tags.py @@ -16,7 +16,7 @@ from collections.abc import Iterable from itertools import product -from domain.api.sds.query import SearchSDSQueryParams +from domain.api.sds.query import SearchSDSDeviceQueryParams def _valid_tag_exists(tag_fields: set[str], data_fields: Iterable[str]) -> bool: @@ -47,7 +47,7 @@ def _generate_all_matching_queries( def sds_metadata_to_device_tags( - data: dict[str, str | Iterable], model: SearchSDSQueryParams + data: dict[str, str | Iterable], model: SearchSDSDeviceQueryParams ) -> list[tuple[tuple[str, str]]]: """ tldr; From 14008d5fdf382970cb58e7901960d97acb383191 Mon Sep 17 00:00:00 2001 From: Joel Klinger Date: Tue, 3 Dec 2024 09:27:44 +0000 Subject: [PATCH 2/2] [feature/PI-618-bulk_etl] fix up tests after tag changes --- src/layers/domain/api/sds/query.py | 6 +- .../tests/test_cpm_system_id_v1.py | 2 +- src/layers/etl_utils/ldif/tests/test_ldif.py | 4 +- .../sds/epr/bulk_create/tests/conftest.py | 2 +- .../epr/bulk_create/tests/test_bulk_create.py | 319 +++--------------- .../bulk_create/tests/test_bulk_repository.py | 18 +- .../tests/test_epr_product_team_repository.py | 21 +- .../sds/epr/bulk_create/tests/test_getters.py | 86 +---- 8 files changed, 90 insertions(+), 368 deletions(-) diff --git a/src/layers/domain/api/sds/query.py b/src/layers/domain/api/sds/query.py index 6dc3fb035..562b9c532 100644 --- a/src/layers/domain/api/sds/query.py +++ b/src/layers/domain/api/sds/query.py @@ -7,9 +7,9 @@ class SearchSDSDeviceQueryParams(BaseModel, extra=Extra.forbid): nhs_mhs_manufacturer_org: str = None nhs_mhs_party_key: str = None - @root_validator - def client_to_id(cls, values): - nhs_as_client = values.get("nhs_as_client") + @root_validator(pre=True) + def client_to_id(cls, values: dict): + nhs_as_client = values.pop("nhs_as_client", None) nhs_id_code = values.get("nhs_id_code") if nhs_as_client and not nhs_id_code: values["nhs_id_code"] = nhs_as_client diff --git a/src/layers/domain/core/cpm_system_id/tests/test_cpm_system_id_v1.py b/src/layers/domain/core/cpm_system_id/tests/test_cpm_system_id_v1.py index 545c6fb00..53872f099 100644 --- a/src/layers/domain/core/cpm_system_id/tests/test_cpm_system_id_v1.py +++ b/src/layers/domain/core/cpm_system_id/tests/test_cpm_system_id_v1.py @@ -19,7 +19,7 @@ def test_party_key_generator_validate_key_valid(): [ "ABC000124", # Missing hyphen "ABC-1234", # Number part too short - "ABC-1234567", # Number part too long + "ABC-123456789101112", # Number part too long "ABC-0001A4", # Number part contains a non-digit character "", # Empty string ], diff --git a/src/layers/etl_utils/ldif/tests/test_ldif.py b/src/layers/etl_utils/ldif/tests/test_ldif.py index 758bf8fc5..f7c902fae 100644 --- a/src/layers/etl_utils/ldif/tests/test_ldif.py +++ b/src/layers/etl_utils/ldif/tests/test_ldif.py @@ -322,7 +322,7 @@ myOtherField: 123 """ -FILTERED_AND_GROUPED_LDIF_TO_FILTER_AND_GROUP_EXAMPLE = """ +FILTERED_AND_GROUPED_LDIF_TO_FILTER_AND_GROUP_EXAMPLE_BASE_64 = """ dn: uniqueIdentifier=AAA1 myField:: QUFB myOtherField: 123 @@ -453,7 +453,7 @@ def test_filter_and_group_ldif_from_s3_by_property_with_b64encoded_group(mocked_ ) assert ( "".join(data.tobytes().decode() for data in filtered_ldif) - == FILTERED_AND_GROUPED_LDIF_TO_FILTER_AND_GROUP_EXAMPLE + == FILTERED_AND_GROUPED_LDIF_TO_FILTER_AND_GROUP_EXAMPLE_BASE_64 ) diff --git a/src/layers/sds/epr/bulk_create/tests/conftest.py b/src/layers/sds/epr/bulk_create/tests/conftest.py index 52375da30..a7b689ff9 100644 --- a/src/layers/sds/epr/bulk_create/tests/conftest.py +++ b/src/layers/sds/epr/bulk_create/tests/conftest.py @@ -100,7 +100,7 @@ def accredited_system_2() -> NhsMhs: nhsrequestorurp="requester-456", nhsdaterequested="a week ago", nhsdateapproved="today", - nhsidcode="AAA", + nhsidcode="BBB", nhsmhspartykey="AAA-456789", nhsproductkey="key-123", nhsasclient={"ABC", "JKL", "LMN"}, diff --git a/src/layers/sds/epr/bulk_create/tests/test_bulk_create.py b/src/layers/sds/epr/bulk_create/tests/test_bulk_create.py index 62caeabca..96e9f9d04 100644 --- a/src/layers/sds/epr/bulk_create/tests/test_bulk_create.py +++ b/src/layers/sds/epr/bulk_create/tests/test_bulk_create.py @@ -88,33 +88,7 @@ def mhs_cpa_ids(): @pytest.fixture def mhs_tags(): - return [ - {"nhs_id_code": "BBB", "nhs_mhs_svc_ia": "my-other-interaction-id"}, - {"nhs_mhs_party_key": "BBB-123456"}, - { - "nhs_id_code": "BBB", - "nhs_mhs_party_key": "BBB-123456", - "nhs_mhs_svc_ia": "my-other-interaction-id", - }, - {"nhs_id_code": "AAA", "nhs_mhs_party_key": "AAA-123456"}, - {"nhs_mhs_svc_ia": "my-interaction-id"}, - {"nhs_mhs_svc_ia": "my-other-interaction-id"}, - {"nhs_id_code": "BBB", "nhs_mhs_party_key": "BBB-123456"}, - {"nhs_id_code": "AAA"}, - {"nhs_mhs_party_key": "AAA-123456", "nhs_mhs_svc_ia": "my-interaction-id"}, - {"nhs_id_code": "BBB"}, - {"nhs_mhs_party_key": "AAA-123456"}, - {"nhs_id_code": "AAA", "nhs_mhs_svc_ia": "my-interaction-id"}, - { - "nhs_mhs_party_key": "BBB-123456", - "nhs_mhs_svc_ia": "my-other-interaction-id", - }, - { - "nhs_id_code": "AAA", - "nhs_mhs_party_key": "AAA-123456", - "nhs_mhs_svc_ia": "my-interaction-id", - }, - ] + return [] @pytest.fixture @@ -166,7 +140,7 @@ def as_device_data(): "Date Approved": "today", "Requestor URP": "requester-456", "Date Requested": "a week ago", - "ODS Code": "AAA", + "ODS Code": "BBB", "MHS Manufacturer Organisation": "AAA", "Party Key": "AAA-456789", "Product Key": "key-123", @@ -179,237 +153,46 @@ def as_device_data(): @pytest.fixture def as_tags(): return [ - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-4", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-4", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "CDE", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "JKL", - "nhs_as_svc_ia": "interaction-id-4", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_manufacturer_org": "AAA", - }, - { - "nhs_as_client": "ABC", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_as_client": "LMN", - "nhs_as_svc_ia": "interaction-id-2", - "nhs_mhs_party_key": "AAA-456789", - }, - { - "nhs_as_client": "EFG", - "nhs_as_svc_ia": "interaction-id-1", - "nhs_mhs_manufacturer_org": "AAA", - }, + [ + { + "nhs_id_code": "AAA", + "nhs_as_svc_ia": "interaction-id-1", + }, + { + "nhs_id_code": "AAA", + "nhs_as_svc_ia": "interaction-id-1", + "nhs_mhs_party_key": "AAA-123456", + }, + { + "nhs_id_code": "AAA", + "nhs_as_svc_ia": "interaction-id-2", + }, + { + "nhs_id_code": "AAA", + "nhs_as_svc_ia": "interaction-id-2", + "nhs_mhs_party_key": "AAA-123456", + }, + ], + [ + { + "nhs_id_code": "BBB", + "nhs_as_svc_ia": "interaction-id-2", + }, + { + "nhs_id_code": "BBB", + "nhs_as_svc_ia": "interaction-id-2", + "nhs_mhs_party_key": "AAA-456789", + }, + { + "nhs_id_code": "BBB", + "nhs_as_svc_ia": "interaction-id-4", + }, + { + "nhs_id_code": "BBB", + "nhs_as_svc_ia": "interaction-id-4", + "nhs_mhs_party_key": "AAA-456789", + }, + ], ] @@ -420,6 +203,12 @@ def _fix_up_questionnaire(qr: QuestionnaireResponse, today_string: str) -> dict: return _qr +def _assert_list_of_dict_equal(a: list[dict], b: list[dict]): + _a = sorted(map(sorted, map(dict.items, a))) + _b = sorted(map(sorted, map(dict.items, b))) + assert _a == _b + + @mock.patch( "sds.epr.bulk_create.bulk_create._create_complete_epr_product", side_effect=lambda **kwargs: kwargs, @@ -501,14 +290,12 @@ def test_create_complete_epr_product__intermediate( assert _message_set_data == message_set_data assert _mhs_data == mhs_data - assert all(tag in kwargs["as_tags"] for tag in as_tags) - assert all(tag in as_tags for tag in kwargs["as_tags"]) - assert len(kwargs["as_tags"]) == len(as_tags) - - assert all(tag in kwargs["mhs_tags"] for tag in mhs_tags) - assert all(tag in mhs_tags for tag in kwargs["mhs_tags"]) - assert len(kwargs["mhs_tags"]) == len(mhs_tags) + expected_tags_1, expected_tags_2 = as_tags + tags_1, tags_2 = kwargs["as_tags"] + _assert_list_of_dict_equal(tags_1, expected_tags_1) + _assert_list_of_dict_equal(tags_2, expected_tags_2) + assert kwargs["mhs_tags"] == mhs_tags assert kwargs["mhs_cpa_ids"] == mhs_cpa_ids assert kwargs["ods_code"] == ods_code assert kwargs["party_key"] == party_key diff --git a/src/layers/sds/epr/bulk_create/tests/test_bulk_repository.py b/src/layers/sds/epr/bulk_create/tests/test_bulk_repository.py index 2a5583b80..e850e10bb 100644 --- a/src/layers/sds/epr/bulk_create/tests/test_bulk_repository.py +++ b/src/layers/sds/epr/bulk_create/tests/test_bulk_repository.py @@ -148,7 +148,10 @@ def test_BulkRepository_handle_ProductTeam(dynamodb_client): ) bulk_repo = BulkRepository(table_name=TABLE_NAME, dynamodb_client=dynamodb_client) - bulk_repo.write([{"ProductTeam": product_team.state()}]) + transactions = bulk_repo.generate_transaction_statements( + {"ProductTeam": product_team.state()} + ) + bulk_repo.write(transactions) product_team_by_id = product_team_repo.read(product_team.id) product_team_by_key = product_team_repo.read("EPR-AAA") @@ -166,7 +169,10 @@ def test_BulkRepository_handle_CpmProduct(dynamodb_client): product.clear_events() bulk_repo = BulkRepository(table_name=TABLE_NAME, dynamodb_client=dynamodb_client) - bulk_repo.write([{"CpmProduct": product.state()}]) + transactions = bulk_repo.generate_transaction_statements( + {"CpmProduct": product.state()} + ) + bulk_repo.write(transactions) product_by_id = product_repo.read(product_team.id, product.id) product_by_key = product_repo.read(product_team.id, "AAA-123456") @@ -186,7 +192,8 @@ def test_BulkRepository_handle_Device(dynamodb_client): device.clear_events() bulk_repo = BulkRepository(table_name=TABLE_NAME, dynamodb_client=dynamodb_client) - bulk_repo.write([{"Device": device.state()}]) + transactions = bulk_repo.generate_transaction_statements({"Device": device.state()}) + bulk_repo.write(transactions) device_by_id = device_repo.read(product_team.id, product.id, device.id) device_by_key = device_repo.read(product_team.id, product.id, "123456") @@ -207,7 +214,10 @@ def test_BulkRepository_handle_DeviceReferenceData(dynamodb_client): device_ref_data = product.create_device_reference_data(name="my-product") bulk_repo = BulkRepository(table_name=TABLE_NAME, dynamodb_client=dynamodb_client) - bulk_repo.write([{"DeviceReferenceData": device_ref_data.state()}]) + transactions = bulk_repo.generate_transaction_statements( + {"DeviceReferenceData": device_ref_data.state()} + ) + bulk_repo.write(transactions) device_ref_data_by_id = device_ref_data_repo.read( product_team.id, product.id, device_ref_data.id diff --git a/src/layers/sds/epr/bulk_create/tests/test_epr_product_team_repository.py b/src/layers/sds/epr/bulk_create/tests/test_epr_product_team_repository.py index cc5f585ca..85b96d0be 100644 --- a/src/layers/sds/epr/bulk_create/tests/test_epr_product_team_repository.py +++ b/src/layers/sds/epr/bulk_create/tests/test_epr_product_team_repository.py @@ -40,9 +40,13 @@ def test_get_all_epr_product_teams_local(dynamodb_client): product_teams = [*epr_product_teams, *non_epr_product_teams] shuffle(product_teams) - BulkRepository(table_name="my-table", dynamodb_client=dynamodb_client).write( - [{"ProductTeam": product_team.state()} for product_team in product_teams] - ) + repo = BulkRepository(table_name="my-table", dynamodb_client=dynamodb_client) + transactions = [] + for product_team in product_teams: + transactions += repo.generate_transaction_statements( + {"ProductTeam": product_team.state()} + ) + repo.write(transactions) epr_product_team_repo = EprProductTeamRepository( table_name="my-table", dynamodb_client=dynamodb_client @@ -76,9 +80,14 @@ def test_get_all_epr_product_teams(): product_teams = [*epr_product_teams, *non_epr_product_teams] shuffle(product_teams) - BulkRepository(table_name=table_name, dynamodb_client=client).write( - [{"ProductTeam": product_team.state()} for product_team in product_teams] - ) + repo = BulkRepository(table_name=table_name, dynamodb_client=client) + transactions = [] + for product_team in product_teams: + transactions += repo.generate_transaction_statements( + {"ProductTeam": product_team.state()} + ) + repo.write(transactions) + epr_product_team_repo = EprProductTeamRepository( table_name=table_name, dynamodb_client=client ) diff --git a/src/layers/sds/epr/bulk_create/tests/test_getters.py b/src/layers/sds/epr/bulk_create/tests/test_getters.py index 551dfaaaa..26cd22632 100644 --- a/src/layers/sds/epr/bulk_create/tests/test_getters.py +++ b/src/layers/sds/epr/bulk_create/tests/test_getters.py @@ -181,102 +181,18 @@ def test_get_additional_interactions_data( def test_get_mhs_tags(mhs_1: NhsMhs, mhs_2: NhsMhs): message_handling_systems = [mhs_1.dict(), mhs_2.dict()] tags = get_mhs_tags(message_handling_systems=message_handling_systems) - - expected_tags = [ - { - "nhs_id_code": "AAA", - }, - { - "nhs_mhs_svc_ia": "my-interaction-id", - }, - { - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_id_code": "AAA", - "nhs_mhs_svc_ia": "my-interaction-id", - }, - { - "nhs_id_code": "AAA", - "nhs_mhs_party_key": "AAA-123456", - }, - { - "nhs_mhs_party_key": "AAA-123456", - "nhs_mhs_svc_ia": "my-interaction-id", - }, - { - "nhs_id_code": "AAA", - "nhs_mhs_party_key": "AAA-123456", - "nhs_mhs_svc_ia": "my-interaction-id", - }, - { - "nhs_id_code": "BBB", - }, - { - "nhs_mhs_svc_ia": "my-other-interaction-id", - }, - { - "nhs_mhs_party_key": "BBB-123456", - }, - { - "nhs_id_code": "BBB", - "nhs_mhs_svc_ia": "my-other-interaction-id", - }, - { - "nhs_id_code": "BBB", - "nhs_mhs_party_key": "BBB-123456", - }, - { - "nhs_mhs_party_key": "BBB-123456", - "nhs_mhs_svc_ia": "my-other-interaction-id", - }, - { - "nhs_id_code": "BBB", - "nhs_mhs_party_key": "BBB-123456", - "nhs_mhs_svc_ia": "my-other-interaction-id", - }, - ] - - for tag in tags: - assert tag in expected_tags, f"{tag} not in expected tags" - - for tag in expected_tags: - assert tag in tags, f"Expected {tag} in tags" - - assert len(tags) == len(expected_tags) + assert len(tags) == 0 def test_get_accredited_system_tags(accredited_system_1: NhsAccreditedSystem): tags = get_accredited_system_tags(accredited_system=accredited_system_1.dict()) expected_tags = [ - { - "nhs_id_code": "AAA", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - "nhs_as_svc_ia": "interaction-id-1", - }, - { - "nhs_id_code": "AAA", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_as_svc_ia": "interaction-id-2", - }, {"nhs_id_code": "AAA", "nhs_as_svc_ia": "interaction-id-1"}, { "nhs_id_code": "AAA", "nhs_mhs_party_key": "AAA-123456", "nhs_as_svc_ia": "interaction-id-1", }, - { - "nhs_id_code": "AAA", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_mhs_party_key": "AAA-123456", - "nhs_as_svc_ia": "interaction-id-2", - }, - { - "nhs_id_code": "AAA", - "nhs_mhs_manufacturer_org": "AAA", - "nhs_as_svc_ia": "interaction-id-1", - }, {"nhs_id_code": "AAA", "nhs_as_svc_ia": "interaction-id-2"}, { "nhs_id_code": "AAA",