further cleanup

ParkenDD · Nov 27, 2024 · 9c053a6 · 9c053a6
1 parent 07bcbd4
commit 9c053a6
Show file tree

Hide file tree

Showing 10 changed files with 61 additions and 38 deletions.
diff --git a/src/parkapi_sources/converters/heidelberg/models.py b/src/parkapi_sources/converters/heidelberg/models.py
@@ -143,7 +143,7 @@ def to_static_parking_site(self) -> StaticParkingSiteInput:
         return StaticParkingSiteInput(
             uid=self.staticParkingSiteId,
             name=self.staticName,
-            description=self.description.replace('\r\n', ' '),
+            description=self.description,
             lat=self.lat,
             lon=self.lon,
             address=f'{self.streetAddress}, {self.postalCode} {self.addressLocality}',

diff --git a/src/parkapi_sources/converters/karlsruhe/models.py b/src/parkapi_sources/converters/karlsruhe/models.py
@@ -3,7 +3,7 @@
 Use of this source code is governed by an MIT-style license that can be found in the LICENSE.txt.
 """
 
-from datetime import date, datetime, time, timezone
+from datetime import datetime, time, timezone
 from decimal import Decimal
 from enum import Enum
 from typing import Optional
@@ -30,7 +30,7 @@
     StaticParkingSiteInput,
 )
 from parkapi_sources.models.enums import OpeningStatus, ParkingSiteType, PurposeType
-from parkapi_sources.validators import MappedBooleanValidator, ParsedDateValidator
+from parkapi_sources.validators import MappedBooleanValidator
 
 
 class KarlsruheOpeningStatus(Enum):
@@ -64,7 +64,7 @@ class KarlsruhePropertiesInput(GeojsonBaseFeaturePropertiesInput):
     betreiber_internet: Optional[str] = Noneable(UrlValidator())
     betreiber_email: Optional[str] = Noneable(EmailValidator())
     betreiber_telefon: Optional[str] = Noneable(StringValidator())
-    stand_stammdaten: date = ParsedDateValidator(date_format='%Y/%m/%d %H:%M:%S')
+    stand_stammdaten: datetime = DateTimeValidator()
 
     def __post_init__(self):
         if self.max_durchfahrtshoehe == 0:  # 0 is used as None

diff --git a/src/parkapi_sources/converters/kienzler/models.py b/src/parkapi_sources/converters/kienzler/models.py
@@ -61,12 +61,15 @@ class KienzlerGeojsonFeaturePropertiesInput(ValidataclassMixin):
     uid: str = StringValidator(min_length=1, max_length=256)
     address: Optional[str] = StringValidator(max_length=512), Default(None)
     type: Optional[ParkingSiteType] = EnumValidator(ParkingSiteType), Default(None)
-    max_height: int = IntegerValidator(min_value=0)
-    max_width: int = IntegerValidator(min_value=0)
-    max_depth: int = IntegerValidator(min_value=0)
+    max_height: int | None = IntegerValidator(min_value=0), Default(None)
+    max_width: int | None = IntegerValidator(min_value=0), Default(None)
+    max_depth: int | None = IntegerValidator(min_value=0), Default(None)
     park_and_ride_type: list[ParkAndRideType] = ListValidator(EnumValidator(ParkAndRideType))
-    external_identifiers: Optional[list[ExternalIdentifierInput]] = ListValidator(
-        DataclassValidator(ExternalIdentifierInput),
+    external_identifiers: list[ExternalIdentifierInput] | None = (
+        ListValidator(
+            DataclassValidator(ExternalIdentifierInput),
+        ),
+        Default(None),
     )
 
     def to_dict(self, **kwargs) -> dict[str, Any]:

diff --git a/src/parkapi_sources/converters/konstanz/models.py b/src/parkapi_sources/converters/konstanz/models.py
@@ -30,6 +30,7 @@
     EmptystringNoneable,
     GermanDurationIntegerValidator,
     MappedBooleanValidator,
+    ReplacingStringValidator,
     TimestampDateTimeValidator,
 )
 
@@ -85,7 +86,7 @@ class KonstanzParkingSiteDataInput:
     has_light: Optional[bool] = EmptystringNoneable(MappedBooleanValidator(mapping={'ja': True, 'nein': False}))
     descript: Optional[str] = EmptystringNoneable(StringValidator())
     public_url: Optional[str] = EmptystringNoneable(UrlValidator())
-    fee_descr: Optional[str] = EmptystringNoneable(StringValidator(multiline=True))
+    fee_descr: Optional[str] = EmptystringNoneable(ReplacingStringValidator(mapping={'\n': ' '}))
     park_ride: Optional[str] = EmptystringNoneable(StringValidator())
     has_live: bool = MappedBooleanValidator(mapping={'ja': True, 'nein': False})
     max_hei: int = EmptystringNoneable(KonstanzHeightValidator())
@@ -106,7 +107,7 @@ def to_static_parking_site(self) -> StaticParkingSiteInput:
             max_stay=self.max_stay,
             max_height=self.max_hei,
             has_lighting=self.has_light,
-            fee_description=None if self.fee_descr is None else self.fee_descr.replace('\n', ' '),
+            fee_description=self.fee_descr,
             has_fee=self.has_fee,
             has_realtime_data=self.has_live,
             lat=self.lat,

diff --git a/src/parkapi_sources/converters/opendata_swiss/models.py b/src/parkapi_sources/converters/opendata_swiss/models.py
@@ -22,6 +22,7 @@
 
 from parkapi_sources.models import GeojsonBaseFeatureInput, RealtimeParkingSiteInput, StaticParkingSiteInput
 from parkapi_sources.models.enums import ParkAndRideType, ParkingSiteType, PurposeType
+from parkapi_sources.validators import ReplacingStringValidator
 
 
 @validataclass
@@ -48,12 +49,21 @@ class OpenDataSwissCapacitiesInput:
     total: int = IntegerValidator()
 
 
+class OpendataSwissReplacingStringValidator(ReplacingStringValidator):
+    def __init__(self, *args, **kwargs):
+        super().__init__(
+            *args,
+            mapping={'\n': ' ', '\r': '', '\xa0': ' ', '\x02': '', '\t': ' '},
+            **kwargs,
+        )
+
+
 @validataclass
 class OpenDataSwissAdditionalInformationInput:
-    de: Optional[str] = Noneable(StringValidator(multiline=False, unsafe=True))
-    en: Optional[str] = Noneable(StringValidator(multiline=False, unsafe=True))
-    it: Optional[str] = Noneable(StringValidator(multiline=False, unsafe=True))
-    fr: Optional[str] = Noneable(StringValidator(multiline=False, unsafe=True))
+    de: Optional[str] = Noneable(OpendataSwissReplacingStringValidator())
+    en: Optional[str] = Noneable(OpendataSwissReplacingStringValidator())
+    it: Optional[str] = Noneable(OpendataSwissReplacingStringValidator())
+    fr: Optional[str] = Noneable(OpendataSwissReplacingStringValidator())
 
 
 class OpenDataSwissOperationTimeDaysOfWeek(Enum):
@@ -120,7 +130,7 @@ class OpenDataSwissPropertiesInput:
     parkingFacilityType: Optional[OpenDataSwissParkingFacilityType] = Noneable(
         EnumValidator(OpenDataSwissParkingFacilityType),
     )
-    salesChannels: Optional[list[str]] = Noneable(ListValidator(StringValidator()))
+    salesChannels: Optional[list[str]] = Noneable(ListValidator(ReplacingStringValidator(mapping={'\n': ' '})))
     operationTime: Optional[OpenDataSwissOperationTimeInput] = Noneable(
         DataclassValidator(OpenDataSwissOperationTimeInput),
     )
@@ -192,9 +202,9 @@ def to_static_parking_site_input(self) -> StaticParkingSiteInput:
         static_parking_site_input.opening_hours = self.properties.get_osm_opening_hours()
 
         if self.properties.additionalInformationForCustomers:
-            static_parking_site_input.description = self.properties.additionalInformationForCustomers.de.replace(
-                '\n', ' '
-            ).replace('\r', ' ')
+            static_parking_site_input.description = self.properties.additionalInformationForCustomers.de
+        else:
+            static_parking_site_input.description = None
 
         if self.properties.address:
             static_parking_site_input.address = f'{self.properties.address.addressLine}, {self.properties.address.postalCode} {self.properties.address.city}'

diff --git a/src/parkapi_sources/converters/pforzheim/converter.py b/src/parkapi_sources/converters/pforzheim/converter.py
@@ -49,12 +49,12 @@ def handle_json(self, data: dict | list) -> tuple[list[StaticParkingSiteInput],
                 type=input_data.type.to_parking_site_type_input(),
                 lat=input_data.lat,
                 lon=input_data.lon,
-                address=input_data.address.replace('\n', ', '),
-                description=input_data.description.replace('\n', ', '),
+                address=input_data.address,
+                description=input_data.description,
                 capacity=input_data.capacity,
                 capacity_woman=input_data.quantitySpacesReservedForWomen,
                 capacity_disabled=input_data.quantitySpacesReservedForMobilityImpededPerson,
-                fee_description=input_data.feeInformation.replace('\n', ', '),
+                fee_description=input_data.feeInformation,
                 supervision_type=SupervisionType.YES if 'ja' in input_data.securityInformation.lower() else False,
                 opening_hours='24/7' if input_data.hasOpeningHours24h else None,
                 static_data_updated_at=datetime.now(tz=timezone.utc),

diff --git a/src/parkapi_sources/converters/pforzheim/validation.py b/src/parkapi_sources/converters/pforzheim/validation.py
@@ -11,7 +11,7 @@
 from validataclass.validators import EnumValidator, IntegerValidator, NumericValidator, StringValidator
 
 from parkapi_sources.models.enums import ParkingSiteType
-from parkapi_sources.validators import ExcelNoneable, MappedBooleanValidator
+from parkapi_sources.validators import ExcelNoneable, MappedBooleanValidator, ReplacingStringValidator
 
 
 class PforzheimParkingSiteType(Enum):
@@ -30,14 +30,14 @@ class PforzheimInput:
     Id: str = StringValidator(max_length=255)
     name: str = StringValidator(max_length=255)
     operatorID: str = StringValidator(max_length=255)
-    address: str = StringValidator(max_length=255, multiline=True)
-    description: str = StringValidator(max_length=512, multiline=True)
+    address: str = ReplacingStringValidator(max_length=255, mapping={'\n': ' '})
+    description: str = ReplacingStringValidator(max_length=512, mapping={'\n': ' '})
     type: PforzheimParkingSiteType = EnumValidator(PforzheimParkingSiteType)
     lat: Decimal = NumericValidator(min_value=40, max_value=60)
     lon: Decimal = NumericValidator(min_value=7, max_value=10)
     capacity: int = IntegerValidator()
     quantitySpacesReservedForWomen: Optional[int] = ExcelNoneable(IntegerValidator())
     quantitySpacesReservedForMobilityImpededPerson: Optional[int] = ExcelNoneable(IntegerValidator())
-    securityInformation: str = StringValidator(multiline=True)
-    feeInformation: str = StringValidator(multiline=True)
+    securityInformation: str = ReplacingStringValidator(mapping={'\n': ' '})
+    feeInformation: str = ReplacingStringValidator(mapping={'\n': ' '})
     hasOpeningHours24h: bool = MappedBooleanValidator(mapping={'wahr': True, 'falsch': False})
diff --git a/src/parkapi_sources/converters/radvis_bw/models.py b/src/parkapi_sources/converters/radvis_bw/models.py
@@ -105,10 +105,13 @@ class RadvisFeaturePropertiesInput:
     gebuehren_pro_monat: Optional[int] = Noneable(IntegerValidator())
     gebuehren_pro_jahr: Optional[int] = Noneable(IntegerValidator())
     beschreibung: Optional[str] = (
-        Noneable(ReplacingStringValidator(multiline=True, mapping={'\x80': ' '})),
+        Noneable(ReplacingStringValidator(mapping={'\x80': ' ', '\n': ' ', '\r': ''})),
+        Default(None),
+    )
+    weitere_information: Optional[str] = (
+        Noneable(ReplacingStringValidator(mapping={'\n': ' ', '\r': ''})),
         Default(None),
     )
-    weitere_information: Optional[str] = Noneable(StringValidator(multiline=True)), Default(None)
     status: StatusType = EnumValidator(StatusType)
 
     def to_dicts(self) -> list[dict]:
@@ -119,8 +122,6 @@ def to_dicts(self) -> list[dict]:
             description = self.beschreibung
         elif self.weitere_information:
             description = self.weitere_information
-        if description is not None:
-            description = description.replace('\r', '').replace('\n', ' ')
 
         base_data = {
             'operator_name': self.betreiber,

diff --git a/src/parkapi_sources/converters/velobrix/converter.py b/src/parkapi_sources/converters/velobrix/converter.py
@@ -29,7 +29,7 @@ class VelobrixPullConverter(PullConverter):
     def get_static_parking_sites(self) -> tuple[list[StaticParkingSiteInput], list[ImportParkingSiteException]]:
         static_parking_site_inputs: list[StaticParkingSiteInput] = []
 
-        velobrix_inputs, import_parking_site_exceptions = self._get_data()
+        velobrix_inputs, import_parking_site_exceptions = self._get_data(include_pricing=True)
 
         for velobrix_input in velobrix_inputs:
             static_parking_site_inputs.append(velobrix_input.to_static_parking_site())
@@ -48,14 +48,17 @@ def get_realtime_parking_sites(self) -> tuple[list[RealtimeParkingSiteInput], li
 
         return realtime_parking_site_inputs, import_parking_site_exceptions
 
-    def _get_data(self) -> tuple[list[VelobrixInput], list[ImportParkingSiteException]]:
+    def _get_data(self, include_pricing: bool = False) -> tuple[list[VelobrixInput], list[ImportParkingSiteException]]:
         velobrix_inputs: list[VelobrixInput] = []
         import_parking_site_exceptions: list[ImportParkingSiteException] = []
 
+        headers = {'Velobrix-ApiKey': self.config_helper.get('PARK_API_VELOBRIX_API_KEY')}
+        if include_pricing:
+            headers['IncludePriceModelDescription'] = 'true'
+
         response = requests.get(
             self.source_info.source_url,
-            params={'api-key': self.config_helper.get('PARK_API_VELOBRIX_API_KEY'), 'limit': 50},
-            headers={'Velobrix-ApiKey': self.config_helper.get('PARK_API_VELOBRIX_API_KEY')},
+            headers=headers,
             timeout=30,
         )
 

diff --git a/src/parkapi_sources/converters/velobrix/models.py b/src/parkapi_sources/converters/velobrix/models.py
@@ -7,7 +7,7 @@
 from datetime import datetime, timezone
 from typing import Optional
 
-from validataclass.dataclasses import validataclass
+from validataclass.dataclasses import Default, validataclass
 from validataclass.validators import (
     AnythingValidator,
     DataclassValidator,
@@ -20,6 +20,7 @@
 )
 
 from parkapi_sources.models import RealtimeParkingSiteInput, StaticParkingSiteInput
+from parkapi_sources.models.enums import PurposeType
 
 
 @validataclass
@@ -54,12 +55,16 @@ class VelobrixInput:
 
     boxTypes: list[VelobrixBoxTypeInput] = ListValidator(DataclassValidator(VelobrixBoxTypeInput))
 
-    priceModelDescription: VelobrixPriceModelDescriptionInput = DataclassValidator(VelobrixPriceModelDescriptionInput)
+    priceModelDescription: VelobrixPriceModelDescriptionInput | None = (
+        Noneable(DataclassValidator(VelobrixPriceModelDescriptionInput)),
+        Default(None),
+    )
 
     def to_static_parking_site(self) -> StaticParkingSiteInput:
         return StaticParkingSiteInput(
             uid=str(self.logicalUnitId),
             name=self.publicName,
+            purpose=PurposeType.BIKE,
             description=' ; '.join([boxType.name for boxType in self.boxTypes]),
             lat=self.locationLat,
             lon=self.locationLon,
@@ -68,7 +73,7 @@ def to_static_parking_site(self) -> StaticParkingSiteInput:
             capacity=self.countLogicalBoxes,
             has_realtime_data=self.countFreeLogicalBoxes is not None,
             has_fee=True,
-            fee_description=self.priceModelDescription.description,
+            fee_description=None if self.priceModelDescription is None else self.priceModelDescription.description,
             static_data_updated_at=datetime.now(timezone.utc),
         )