From 3c867ac6820d3164ff60ea649fc349de6e92fbba Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Wed, 24 Jan 2024 09:53:00 -0700 Subject: [PATCH 1/7] Fix for JSON on optional nested types. --- pydantic_settings/sources.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 335ba9dd..e9a5b07b 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -495,7 +495,7 @@ def _field_is_complex(self, field: FieldInfo) -> tuple[bool, bool]: return True, allow_parse_failure @staticmethod - def next_field(field: FieldInfo | None, key: str) -> FieldInfo | None: + def next_field(field: FieldInfo | Any | None, key: str) -> FieldInfo | None: """ Find the field in a sub model by key(env name) @@ -524,11 +524,25 @@ class Cfg(BaseSettings): Returns: Field if it finds the next field otherwise `None`. """ - if not field or origin_is_union(get_origin(field.annotation)): - # no support for Unions of complex BaseSettings fields + if not field: return None - elif field.annotation and hasattr(field.annotation, 'model_fields') and field.annotation.model_fields.get(key): - return field.annotation.model_fields[key] + if isinstance(field, FieldInfo): + if not hasattr(field, 'annotation'): + return None + annotation = field.annotation + else: + annotation = field + + if origin_is_union(get_origin(annotation)) or isinstance(annotation, WithArgsTypes): + type_ = get_origin(annotation) + if is_model_class(type_) and type_.model_fields.get(key): + return type_.model_fields.get(key) + for type_ in get_args(annotation): + type_has_key = EnvSettingsSource.next_field(type_, key) + if type_has_key: + return type_has_key + elif is_model_class(annotation) and annotation.model_fields.get(key): + return annotation.model_fields[key] return None From 0102ad65bd66fb019bb574b5785388ae66145f3c Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Wed, 24 Jan 2024 09:58:08 -0700 Subject: [PATCH 2/7] Remove unused case. --- pydantic_settings/sources.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index e9a5b07b..ae40ba1c 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -11,8 +11,8 @@ from dotenv import dotenv_values from pydantic import AliasChoices, AliasPath, BaseModel, Json, TypeAdapter -from pydantic._internal._typing_extra import origin_is_union -from pydantic._internal._utils import deep_update, lenient_issubclass +from pydantic._internal._typing_extra import WithArgsTypes, origin_is_union +from pydantic._internal._utils import deep_update, is_model_class, lenient_issubclass from pydantic.fields import FieldInfo from typing_extensions import get_args, get_origin @@ -534,9 +534,6 @@ class Cfg(BaseSettings): annotation = field if origin_is_union(get_origin(annotation)) or isinstance(annotation, WithArgsTypes): - type_ = get_origin(annotation) - if is_model_class(type_) and type_.model_fields.get(key): - return type_.model_fields.get(key) for type_ in get_args(annotation): type_has_key = EnvSettingsSource.next_field(type_, key) if type_has_key: From 6ead8d2b1091ecd21a8405cac3daa5c94cdf6454 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Wed, 24 Jan 2024 10:13:52 -0700 Subject: [PATCH 3/7] Add test. --- pydantic_settings/sources.py | 7 +------ tests/test_settings.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index ae40ba1c..6290fba6 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -526,13 +526,8 @@ class Cfg(BaseSettings): """ if not field: return None - if isinstance(field, FieldInfo): - if not hasattr(field, 'annotation'): - return None - annotation = field.annotation - else: - annotation = field + annotation = field.annotation if isinstance(field, FieldInfo) else field if origin_is_union(get_origin(annotation)) or isinstance(annotation, WithArgsTypes): for type_ in get_args(annotation): type_has_key = EnvSettingsSource.next_field(type_, key) diff --git a/tests/test_settings.py b/tests/test_settings.py index b2371953..9f194608 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -198,6 +198,22 @@ class Cfg(BaseSettings): } +def test_nested_optional_json(env): + class Child(BaseModel): + num_list: Optional[List[int]] = None + + class Cfg(BaseSettings, env_nested_delimiter='__'): + child: Optional[Child] = None + + env.set('CHILD__NUM_LIST', '[1,2,3]') + cfg = Cfg() + assert cfg.model_dump() == { + 'child': { + 'num_list': [1, 2, 3], + }, + } + + def test_nested_env_delimiter_with_prefix(env): class Subsettings(BaseSettings): banana: str From ec32e4f7ea668494a1e1119d805e3f702b5d27bd Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 25 Jan 2024 10:16:08 -0700 Subject: [PATCH 4/7] Fix and add test case complex union. --- pydantic_settings/sources.py | 11 +++++++---- tests/test_settings.py | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/pydantic_settings/sources.py b/pydantic_settings/sources.py index 6290fba6..c9a32430 100644 --- a/pydantic_settings/sources.py +++ b/pydantic_settings/sources.py @@ -188,6 +188,8 @@ def _extract_field_info(self, field: FieldInfo, field_name: str) -> list[tuple[s ) else: # string validation alias field_info.append((v_alias, self._apply_case_sensitive(v_alias), False)) + elif origin_is_union(get_origin(field.annotation)) and _union_is_complex(field.annotation, field.metadata): + field_info.append((field_name, self._apply_case_sensitive(self.env_prefix + field_name), True)) else: field_info.append((field_name, self._apply_case_sensitive(self.env_prefix + field_name), False)) @@ -478,16 +480,13 @@ def prepare_field_value(self, field_name: str, field: FieldInfo, value: Any, val # simplest case, field is not complex, we only need to add the value if it was found return value - def _union_is_complex(self, annotation: type[Any] | None, metadata: list[Any]) -> bool: - return any(_annotation_is_complex(arg, metadata) for arg in get_args(annotation)) - def _field_is_complex(self, field: FieldInfo) -> tuple[bool, bool]: """ Find out if a field is complex, and if so whether JSON errors should be ignored """ if self.field_is_complex(field): allow_parse_failure = False - elif origin_is_union(get_origin(field.annotation)) and self._union_is_complex(field.annotation, field.metadata): + elif origin_is_union(get_origin(field.annotation)) and _union_is_complex(field.annotation, field.metadata): allow_parse_failure = True else: return False, False @@ -722,3 +721,7 @@ def _annotation_is_complex_inner(annotation: type[Any] | None) -> bool: return lenient_issubclass(annotation, (BaseModel, Mapping, Sequence, tuple, set, frozenset, deque)) or is_dataclass( annotation ) + + +def _union_is_complex(annotation: type[Any] | None, metadata: list[Any]) -> bool: + return any(_annotation_is_complex(arg, metadata) for arg in get_args(annotation)) diff --git a/tests/test_settings.py b/tests/test_settings.py index 9f194608..d66908ad 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -198,7 +198,7 @@ class Cfg(BaseSettings): } -def test_nested_optional_json(env): +def test_nested_env_optional_json(env): class Child(BaseModel): num_list: Optional[List[int]] = None @@ -1228,6 +1228,19 @@ class Settings(BaseSettings): assert Settings().model_dump() == {'foo': {'a': 'b'}} +def test_secrets_nested_optional_json(tmp_path): + class Foo(BaseModel): + a: int + + class Settings(BaseSettings): + foo: Foo | None = None + + p = tmp_path / 'foo' + p.write_text('{"a": 10}') + + assert Settings(_secrets_dir=tmp_path).model_dump() == {'foo': {'a': 10}} + + def test_secrets_path_invalid_json(tmp_path): p = tmp_path / 'foo' p.write_text('{"a": "b"') From e31311f31a30d75035a1d01e3f0c47859fc5888b Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 25 Jan 2024 10:25:09 -0700 Subject: [PATCH 5/7] Python 3.8 fixes. --- tests/test_settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index d66908ad..52b242b0 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1233,7 +1233,7 @@ class Foo(BaseModel): a: int class Settings(BaseSettings): - foo: Foo | None = None + foo: Optional[Foo] = None p = tmp_path / 'foo' p.write_text('{"a": 10}') From 7166825dc228e57fa2ae0228efb32e74f4140674 Mon Sep 17 00:00:00 2001 From: Kyle Schwab Date: Thu, 25 Jan 2024 20:02:13 -0700 Subject: [PATCH 6/7] Remove stale documentation. --- docs/index.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/index.md b/docs/index.md index d1627c9d..a05a2bce 100644 --- a/docs/index.md +++ b/docs/index.md @@ -324,9 +324,6 @@ print(Settings().model_dump()) `env_nested_delimiter` can be configured via the `model_config` as shown above, or via the `_env_nested_delimiter` keyword argument on instantiation. -JSON is only parsed in top-level fields, if you need to parse JSON in sub-models, you will need to implement -validators on those models. - Nested environment variables take precedence over the top-level environment variable JSON (e.g. in the example above, `SUB_MODEL__V2` trumps `SUB_MODEL`). From 6bbd33231cc0a8580fa45d1c5b7dd0fe3aeb6b44 Mon Sep 17 00:00:00 2001 From: Hasan Ramezani Date: Fri, 26 Jan 2024 16:09:23 +0100 Subject: [PATCH 7/7] Update tests/test_settings.py --- tests/test_settings.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_settings.py b/tests/test_settings.py index 52b242b0..a7d6912b 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1229,16 +1229,18 @@ class Settings(BaseSettings): def test_secrets_nested_optional_json(tmp_path): + p = tmp_path / 'foo' + p.write_text('{"a": 10}') + class Foo(BaseModel): a: int class Settings(BaseSettings): foo: Optional[Foo] = None - p = tmp_path / 'foo' - p.write_text('{"a": 10}') + model_config = SettingsConfigDict(secrets_dir=tmp_path) - assert Settings(_secrets_dir=tmp_path).model_dump() == {'foo': {'a': 10}} + assert Settings().model_dump() == {'foo': {'a': 10}} def test_secrets_path_invalid_json(tmp_path):